71 enum KindTy { Token, Immediate, Register, Expression } Kind;
73 SMLoc StartLoc, EndLoc;
74 const AMDGPUAsmParser *AsmParser;
77 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
78 : Kind(Kind_), AsmParser(AsmParser_) {}
80 using Ptr = std::unique_ptr<AMDGPUOperand>;
88 bool hasFPModifiers()
const {
return Abs || Neg; }
89 bool hasIntModifiers()
const {
return Sext; }
90 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
91 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
92 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
94 int64_t getFPModifiersOperand()
const {
101 int64_t getIntModifiersOperand()
const {
107 int64_t getModifiersOperand()
const {
108 assert(!(hasFPModifiers() && hasIntModifiers()) &&
109 "fp and int modifiers should not be used simultaneously");
110 if (hasFPModifiers())
111 return getFPModifiersOperand();
112 if (hasIntModifiers())
113 return getIntModifiersOperand();
117 friend raw_ostream &
operator<<(raw_ostream &OS,
118 AMDGPUOperand::Modifiers Mods);
192 ImmTyMatrixAScaleFmt,
193 ImmTyMatrixBScaleFmt,
226 mutable int MCOpIdx = -1;
229 bool isToken()
const override {
return Kind == Token; }
231 bool isSymbolRefExpr()
const {
235 bool isImm()
const override {
return Kind == Immediate; }
237 bool isInlinableImm(MVT type)
const;
238 bool isLiteralImm(MVT type)
const;
240 bool isRegKind()
const {
return Kind == Register; }
242 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
244 bool isRegOrInline(
unsigned RCID, MVT type)
const {
245 return isRegClass(RCID) || isInlinableImm(type);
249 return isRegOrInline(RCID, type) || isLiteralImm(type);
252 bool isRegOrImmWithInt16InputMods()
const {
256 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
258 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
261 bool isRegOrImmWithInt32InputMods()
const {
265 bool isRegOrInlineImmWithInt16InputMods()
const {
266 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
269 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
270 return isRegOrInline(
271 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
274 bool isRegOrInlineImmWithInt32InputMods()
const {
275 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
278 bool isRegOrImmWithInt64InputMods()
const {
282 bool isRegOrImmWithFP16InputMods()
const {
286 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
288 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
291 bool isRegOrImmWithFP32InputMods()
const {
295 bool isRegOrImmWithFP64InputMods()
const {
299 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
300 return isRegOrInline(
301 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
304 bool isRegOrInlineImmWithFP32InputMods()
const {
305 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
308 bool isRegOrInlineImmWithFP64InputMods()
const {
309 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
312 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
314 bool isVRegWithFP32InputMods()
const {
315 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
318 bool isVRegWithFP64InputMods()
const {
319 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
322 bool isPackedFP16InputMods()
const {
326 bool isPackedVGPRFP32InputMods()
const {
330 bool isVReg()
const {
331 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
332 isRegClass(AMDGPU::VReg_64RegClassID) ||
333 isRegClass(AMDGPU::VReg_96RegClassID) ||
334 isRegClass(AMDGPU::VReg_128RegClassID) ||
335 isRegClass(AMDGPU::VReg_160RegClassID) ||
336 isRegClass(AMDGPU::VReg_192RegClassID) ||
337 isRegClass(AMDGPU::VReg_256RegClassID) ||
338 isRegClass(AMDGPU::VReg_512RegClassID) ||
339 isRegClass(AMDGPU::VReg_1024RegClassID);
342 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
344 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
346 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
348 bool isAV_LdSt_32_Align2_RegOp()
const {
349 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
350 isRegClass(AMDGPU::AGPR_32RegClassID);
353 bool isVRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
355 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
357 bool isSDWAOperand(MVT type)
const;
358 bool isSDWAFP16Operand()
const;
359 bool isSDWAFP32Operand()
const;
360 bool isSDWAInt16Operand()
const;
361 bool isSDWAInt32Operand()
const;
363 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
365 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
367 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
369 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
371 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
372 bool isDim()
const {
return isImmTy(ImmTyDim); }
373 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
374 bool isOff()
const {
return isImmTy(ImmTyOff); }
375 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
376 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
377 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
378 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
379 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
380 bool isFlatOffset()
const {
381 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
383 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
384 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
385 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
386 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
387 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
388 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
389 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
390 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
391 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
392 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
393 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
394 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
395 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
396 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
397 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
398 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
399 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
400 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
401 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
402 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
403 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
404 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
405 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
406 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
407 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
408 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
409 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
410 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
411 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
412 bool isDone()
const {
return isImmTy(ImmTyDone); }
413 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
415 bool isRegOrImm()
const {
return isReg() || isImm(); }
417 bool isRegClass(
unsigned RCID)
const;
421 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
422 return isRegOrInline(RCID, type) && !hasModifiers();
425 bool isSCSrcB16()
const {
426 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
429 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
431 bool isSCSrc_b32()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
435 bool isSCSrc_b64()
const {
436 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
439 bool isBoolReg()
const;
441 bool isSCSrcF16()
const {
442 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
445 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
447 bool isSCSrcF32()
const {
448 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
451 bool isSCSrcF64()
const {
452 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
455 bool isSSrc_b32()
const {
456 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
459 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
461 bool isSSrcV2B16()
const {
466 bool isSSrc_b64()
const {
469 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
470 (((
const MCTargetAsmParser *)AsmParser)
471 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
475 bool isSSrc_f32()
const {
476 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
479 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
481 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
483 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
485 bool isSSrcV2F16()
const {
490 bool isSSrcV2FP32()
const {
495 bool isSCSrcV2FP32()
const {
500 bool isSSrcV2INT32()
const {
505 bool isSCSrcV2INT32()
const {
507 return isSCSrc_b32();
510 bool isSSrcOrLds_b32()
const {
511 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
512 isLiteralImm(MVT::i32) || isExpr();
515 bool isVCSrc_b32()
const {
516 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
519 bool isVCSrc_b32_Lo256()
const {
520 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
523 bool isVCSrc_b64_Lo256()
const {
524 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
527 bool isVCSrc_b64()
const {
528 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
531 bool isVCSrcT_b16()
const {
532 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
535 bool isVCSrcTB16_Lo128()
const {
536 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
539 bool isVCSrcFake16B16_Lo128()
const {
540 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
543 bool isVCSrc_b16()
const {
544 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
547 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
549 bool isVCSrc_f32()
const {
550 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
553 bool isVCSrc_f64()
const {
554 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
557 bool isVCSrcTBF16()
const {
558 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
561 bool isVCSrcT_f16()
const {
562 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
565 bool isVCSrcT_bf16()
const {
566 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
569 bool isVCSrcTBF16_Lo128()
const {
570 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
573 bool isVCSrcTF16_Lo128()
const {
574 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
577 bool isVCSrcFake16BF16_Lo128()
const {
578 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
581 bool isVCSrcFake16F16_Lo128()
const {
582 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
585 bool isVCSrc_bf16()
const {
586 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
589 bool isVCSrc_f16()
const {
590 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
593 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
595 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
597 bool isVSrc_b32()
const {
598 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
601 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
603 bool isVSrc_v2b64()
const {
604 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
605 isLiteralImm(MVT::i64);
608 bool isVSrc_v2f64()
const {
609 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
610 isLiteralImm(MVT::f64);
613 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
615 bool isVSrcT_b16_Lo128()
const {
616 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
619 bool isVSrcFake16_b16_Lo128()
const {
620 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
623 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
625 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
627 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
629 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
631 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
633 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
635 bool isVSrc_f32()
const {
636 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
639 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
641 bool isVSrcT_bf16()
const {
642 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
645 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
647 bool isVSrcT_bf16_Lo128()
const {
648 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
651 bool isVSrcT_f16_Lo128()
const {
652 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
655 bool isVSrcFake16_bf16_Lo128()
const {
656 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
659 bool isVSrcFake16_f16_Lo128()
const {
660 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
663 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
665 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
667 bool isVSrc_v2bf16()
const {
668 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
671 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
673 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
675 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
677 bool isVISrcB32()
const {
678 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
681 bool isVISrcB16()
const {
682 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
685 bool isVISrcV2B16()
const {
return isVISrcB16(); }
687 bool isVISrcF32()
const {
688 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
691 bool isVISrcF16()
const {
692 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
695 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
697 bool isVISrc_64_bf16()
const {
698 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
701 bool isVISrc_64_f16()
const {
702 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
705 bool isVISrc_64_b32()
const {
706 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
709 bool isVISrc_64B64()
const {
710 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
713 bool isVISrc_64_f64()
const {
714 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
717 bool isVISrc_64V2FP32()
const {
718 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
721 bool isVISrc_64V2INT32()
const {
722 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
725 bool isVISrc_256_b32()
const {
726 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
729 bool isVISrc_256_f32()
const {
730 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
733 bool isVISrc_256B64()
const {
734 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
737 bool isVISrc_256_f64()
const {
738 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
741 bool isVISrc_512_f64()
const {
742 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
745 bool isVISrc_128B16()
const {
746 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
749 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
751 bool isVISrc_128_b32()
const {
752 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
755 bool isVISrc_128_f32()
const {
756 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
759 bool isVISrc_256V2FP32()
const {
760 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
763 bool isVISrc_256V2INT32()
const {
764 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
767 bool isVISrc_512_b32()
const {
768 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
771 bool isVISrc_512B16()
const {
772 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
775 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
777 bool isVISrc_512_f32()
const {
778 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
781 bool isVISrc_512F16()
const {
782 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
785 bool isVISrc_512V2F16()
const {
786 return isVISrc_512F16() || isVISrc_512_b32();
789 bool isVISrc_1024_b32()
const {
790 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
793 bool isVISrc_1024B16()
const {
794 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
797 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
799 bool isVISrc_1024_f32()
const {
800 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
803 bool isVISrc_1024F16()
const {
804 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
807 bool isVISrc_1024V2F16()
const {
808 return isVISrc_1024F16() || isVISrc_1024_b32();
811 bool isAISrcB32()
const {
812 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
815 bool isAISrcB16()
const {
816 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
819 bool isAISrcV2B16()
const {
return isAISrcB16(); }
821 bool isAISrcF32()
const {
822 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
825 bool isAISrcF16()
const {
826 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
829 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
831 bool isAISrc_64B64()
const {
832 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
835 bool isAISrc_64_f64()
const {
836 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
839 bool isAISrc_128_b32()
const {
840 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
843 bool isAISrc_128B16()
const {
844 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
847 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
849 bool isAISrc_128_f32()
const {
850 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
853 bool isAISrc_128F16()
const {
854 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
857 bool isAISrc_128V2F16()
const {
858 return isAISrc_128F16() || isAISrc_128_b32();
861 bool isVISrc_128_bf16()
const {
862 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
865 bool isVISrc_128_f16()
const {
866 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
869 bool isVISrc_128V2F16()
const {
870 return isVISrc_128_f16() || isVISrc_128_b32();
873 bool isAISrc_256B64()
const {
874 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
877 bool isAISrc_256_f64()
const {
878 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
881 bool isAISrc_512_b32()
const {
882 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
885 bool isAISrc_512B16()
const {
886 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
889 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
891 bool isAISrc_512_f32()
const {
892 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
895 bool isAISrc_512F16()
const {
896 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
899 bool isAISrc_512V2F16()
const {
900 return isAISrc_512F16() || isAISrc_512_b32();
903 bool isAISrc_1024_b32()
const {
904 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
907 bool isAISrc_1024B16()
const {
908 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
911 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
913 bool isAISrc_1024_f32()
const {
914 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
917 bool isAISrc_1024F16()
const {
918 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
921 bool isAISrc_1024V2F16()
const {
922 return isAISrc_1024F16() || isAISrc_1024_b32();
925 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
927 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
929 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
931 bool isMem()
const override {
return false; }
933 bool isExpr()
const {
return Kind == Expression; }
935 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
937 bool isSWaitCnt()
const;
938 bool isDepCtr()
const;
939 bool isSDelayALU()
const;
940 bool isHwreg()
const;
941 bool isSendMsg()
const;
942 bool isWaitEvent()
const;
943 bool isSplitBarrier()
const;
944 bool isSwizzle()
const;
945 bool isSMRDOffset8()
const;
946 bool isSMEMOffset()
const;
947 bool isSMRDLiteralOffset()
const;
949 bool isDPPCtrl()
const;
951 bool isGPRIdxMode()
const;
952 bool isS16Imm()
const;
953 bool isU16Imm()
const;
954 bool isEndpgm()
const;
956 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
957 return [
this,
P]() {
return P(*
this); };
962 return StringRef(Tok.Data, Tok.Length);
970 void setImm(int64_t Val) {
975 ImmTy getImmTy()
const {
980 MCRegister
getReg()
const override {
985 SMLoc getStartLoc()
const override {
return StartLoc; }
987 SMLoc getEndLoc()
const override {
return EndLoc; }
989 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
991 int getMCOpIdx()
const {
return MCOpIdx; }
993 Modifiers getModifiers()
const {
994 assert(isRegKind() || isImmTy(ImmTyNone));
995 return isRegKind() ?
Reg.Mods :
Imm.Mods;
998 void setModifiers(Modifiers Mods) {
999 assert(isRegKind() || isImmTy(ImmTyNone));
1006 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1008 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1010 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1012 bool isForcedLit()
const {
1013 return isImmLiteral() && getModifiers().isForcedLit();
1016 bool isForcedLit64()
const {
1017 return isImmLiteral() && getModifiers().isForcedLit64();
1022 void addImmOperands(MCInst &Inst,
unsigned N,
1023 bool ApplyModifiers =
true)
const;
1025 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1026 bool ApplyModifiers)
const;
1028 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1030 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1032 addRegOperands(Inst,
N);
1034 addImmOperands(Inst,
N);
1037 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1038 Modifiers Mods = getModifiers();
1041 addRegOperands(Inst,
N);
1043 addImmOperands(Inst,
N,
false);
1047 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1048 assert(!hasIntModifiers());
1049 addRegOrImmWithInputModsOperands(Inst,
N);
1052 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1053 assert(!hasFPModifiers());
1054 addRegOrImmWithInputModsOperands(Inst,
N);
1057 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1058 Modifiers Mods = getModifiers();
1061 addRegOperands(Inst,
N);
1064 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1065 assert(!hasIntModifiers());
1066 addRegWithInputModsOperands(Inst,
N);
1069 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1070 assert(!hasFPModifiers());
1071 addRegWithInputModsOperands(Inst,
N);
1074 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1077 case ImmTyNone: OS <<
"None";
break;
1078 case ImmTyGDS: OS <<
"GDS";
break;
1079 case ImmTyLDS: OS <<
"LDS";
break;
1080 case ImmTyOffen: OS <<
"Offen";
break;
1081 case ImmTyIdxen: OS <<
"Idxen";
break;
1082 case ImmTyAddr64: OS <<
"Addr64";
break;
1083 case ImmTyOffset: OS <<
"Offset";
break;
1084 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1085 case ImmTyOffset0: OS <<
"Offset0";
break;
1086 case ImmTyOffset1: OS <<
"Offset1";
break;
1087 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1088 case ImmTyCPol: OS <<
"CPol";
break;
1089 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1091 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1092 case ImmTyTFE: OS <<
"TFE";
break;
1093 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1094 case ImmTyD16: OS <<
"D16";
break;
1095 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1096 case ImmTyClamp: OS <<
"Clamp";
break;
1097 case ImmTyOModSI: OS <<
"OModSI";
break;
1098 case ImmTyDPP8: OS <<
"DPP8";
break;
1099 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1100 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1101 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1102 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1103 case ImmTyDppFI: OS <<
"DppFI";
break;
1104 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1105 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1106 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1107 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1108 case ImmTyDMask: OS <<
"DMask";
break;
1109 case ImmTyDim: OS <<
"Dim";
break;
1110 case ImmTyUNorm: OS <<
"UNorm";
break;
1111 case ImmTyDA: OS <<
"DA";
break;
1112 case ImmTyR128A16: OS <<
"R128A16";
break;
1113 case ImmTyA16: OS <<
"A16";
break;
1114 case ImmTyLWE: OS <<
"LWE";
break;
1115 case ImmTyOff: OS <<
"Off";
break;
1116 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1117 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1118 case ImmTyExpVM: OS <<
"ExpVM";
break;
1119 case ImmTyDone: OS <<
"Done";
break;
1120 case ImmTyRowEn: OS <<
"RowEn";
break;
1121 case ImmTyHwreg: OS <<
"Hwreg";
break;
1122 case ImmTySendMsg: OS <<
"SendMsg";
break;
1123 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1124 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1125 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1126 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1127 case ImmTyOpSel: OS <<
"OpSel";
break;
1128 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1129 case ImmTyNegLo: OS <<
"NegLo";
break;
1130 case ImmTyNegHi: OS <<
"NegHi";
break;
1131 case ImmTySwizzle: OS <<
"Swizzle";
break;
1132 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1133 case ImmTyHigh: OS <<
"High";
break;
1134 case ImmTyBLGP: OS <<
"BLGP";
break;
1135 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1136 case ImmTyABID: OS <<
"ABID";
break;
1137 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1138 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1139 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1140 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1141 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1142 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1143 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1144 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1145 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1146 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1147 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1148 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1149 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1150 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1151 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1152 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1157 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1161 <<
" mods: " <<
Reg.Mods <<
'>';
1165 if (getImmTy() != ImmTyNone) {
1167 printImmTy(OS, getImmTy());
1169 OS <<
" mods: " <<
Imm.Mods <<
'>';
1182 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1183 int64_t Val, SMLoc Loc,
1184 ImmTy
Type = ImmTyNone,
1185 bool IsFPImm =
false) {
1186 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1188 Op->Imm.IsFPImm = IsFPImm;
1190 Op->Imm.Mods = Modifiers();
1196 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1197 StringRef Str, SMLoc Loc,
1198 bool HasExplicitEncodingSize =
true) {
1199 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1200 Res->Tok.Data = Str.data();
1201 Res->Tok.Length = Str.size();
1202 Res->StartLoc = Loc;
1207 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1208 MCRegister
Reg, SMLoc S, SMLoc
E) {
1209 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1210 Op->Reg.RegNo =
Reg;
1211 Op->Reg.Mods = Modifiers();
1217 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1218 const class MCExpr *Expr, SMLoc S) {
1219 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1228 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1237#define GET_REGISTER_MATCHER
1238#include "AMDGPUGenAsmMatcher.inc"
1239#undef GET_REGISTER_MATCHER
1240#undef GET_SUBTARGET_FEATURE_NAME
1245class KernelScopeInfo {
1246 int SgprIndexUnusedMin = -1;
1247 int VgprIndexUnusedMin = -1;
1248 int AgprIndexUnusedMin = -1;
1252 void usesSgprAt(
int i) {
1253 if (i >= SgprIndexUnusedMin) {
1254 SgprIndexUnusedMin = ++i;
1257 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1263 void usesVgprAt(
int i) {
1264 if (i >= VgprIndexUnusedMin) {
1265 VgprIndexUnusedMin = ++i;
1268 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1270 VgprIndexUnusedMin);
1276 void usesAgprAt(
int i) {
1281 if (i >= AgprIndexUnusedMin) {
1282 AgprIndexUnusedMin = ++i;
1285 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1290 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1292 VgprIndexUnusedMin);
1299 KernelScopeInfo() =
default;
1303 MSTI = Ctx->getSubtargetInfo();
1305 usesSgprAt(SgprIndexUnusedMin = -1);
1306 usesVgprAt(VgprIndexUnusedMin = -1);
1308 usesAgprAt(AgprIndexUnusedMin = -1);
1312 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1313 unsigned RegWidth) {
1316 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1319 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1322 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1331 MCAsmParser &Parser;
1333 unsigned ForcedEncodingSize = 0;
1334 bool ForcedDPP =
false;
1335 bool ForcedSDWA =
false;
1336 KernelScopeInfo KernelScope;
1337 const unsigned HwMode;
1339 const AMDGPU::IsaVersion ISA;
1344#define GET_ASSEMBLER_HEADER
1345#include "AMDGPUGenAsmMatcher.inc"
1350 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1352 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1356 std::optional<AMDGPU::InfoSectionData> InfoData;
1363 bool TargetDirectiveEmitted =
false;
1372 SmallVector<unsigned> OpcodeStream;
1374 OpcodeStreamSymbols;
1375 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1378 void checkKernelPrologues();
1381 void createConstantSymbol(StringRef Id, int64_t Val);
1383 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1384 bool OutOfRangeError(SMRange
Range);
1400 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1401 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1402 std::optional<bool> EnableWavefrontSize32,
1403 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1404 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1405 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1406 bool ParseDirectiveAMDGCNTarget();
1407 bool ParseDirectiveAMDHSACodeObjectVersion();
1408 bool ParseDirectiveAMDHSAKernel();
1409 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1410 bool ParseDirectiveAMDKernelCodeT();
1412 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1413 bool ParseDirectiveAMDGPUHsaKernel();
1415 bool ParseDirectiveISAVersion();
1416 bool ParseDirectiveHSAMetadata();
1417 bool ParseDirectivePALMetadataBegin();
1418 bool ParseDirectivePALMetadata();
1419 bool ParseDirectiveAMDGPULDS();
1420 bool ParseDirectiveAMDGPUInfo();
1424 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1425 const char *AssemblerDirectiveEnd,
1426 std::string &CollectString);
1428 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1429 RegisterKind RegKind, MCRegister Reg1,
1430 RegisterKind RegKind1, SMLoc Loc);
1431 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1432 unsigned &RegNum,
unsigned &RegWidth,
1433 bool RestoreOnFailure =
false);
1434 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1435 unsigned &RegNum,
unsigned &RegWidth,
1436 SmallVectorImpl<AsmToken> &Tokens);
1437 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1439 SmallVectorImpl<AsmToken> &Tokens);
1440 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1442 SmallVectorImpl<AsmToken> &Tokens);
1443 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1445 SmallVectorImpl<AsmToken> &Tokens);
1446 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1447 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1448 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1451 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1452 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1453 void initializeGprCountSymbol(RegisterKind RegKind);
1454 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1460 OperandMode_Default,
1464 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1466 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1467 const MCInstrInfo &MII)
1468 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1469 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1474 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1476 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1477 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1478 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1479 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1481 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1482 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1483 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1485 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1486 initializeGprCountSymbol(IS_VGPR);
1487 initializeGprCountSymbol(IS_SGPR);
1492 createConstantSymbol(Symbol, Code);
1494 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1495 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1496 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1544 bool hasBVHRayTracingInsts()
const {
1545 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1550 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1552 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1554 bool hasInv2PiInlineImm()
const {
1555 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1558 bool has64BitLiterals()
const {
1559 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1562 bool hasFlatOffsets()
const {
1563 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1566 bool hasTrue16Insts()
const {
1567 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1571 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1574 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1576 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1578 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1580 bool hasPartialNSAEncoding()
const {
1581 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1584 bool hasGloballyAddressableScratch()
const {
1585 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1598 AMDGPUTargetStreamer &getTargetStreamer() {
1599 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1600 return static_cast<AMDGPUTargetStreamer &
>(TS);
1606 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1609 const MCRegisterInfo *getMRI()
const {
1613 const MCInstrInfo *getMII()
const {
return &MII; }
1617 const FeatureBitset &getFeatureBits()
const {
1618 return getSTI().getFeatureBits();
1621 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1622 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1623 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1625 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1626 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1627 bool isForcedDPP()
const {
return ForcedDPP; }
1628 bool isForcedSDWA()
const {
return ForcedSDWA; }
1629 ArrayRef<unsigned> getMatchedVariants()
const;
1630 StringRef getMatchedVariantName()
const;
1632 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1633 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1634 bool RestoreOnFailure);
1635 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1636 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1637 SMLoc &EndLoc)
override;
1638 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1639 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1640 unsigned Kind)
override;
1641 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1644 bool MatchingInlineAsm)
override;
1645 bool ParseDirective(AsmToken DirectiveID)
override;
1646 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1647 void onEndOfFile()
override;
1649 OperandMode
Mode = OperandMode_Default);
1650 StringRef parseMnemonicSuffix(StringRef Name);
1651 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1657 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1661 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1662 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1664 ParseStatus parseOperandArrayWithPrefix(
1666 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1667 bool (*ConvertResult)(int64_t &) =
nullptr);
1671 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1672 bool IgnoreNegative =
false);
1673 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1677 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1681 ArrayRef<const char *> Ids,
1685 ArrayRef<const char *> Ids,
1686 AMDGPUOperand::ImmTy
Type);
1689 bool isOperandModifier(
const AsmToken &Token,
1690 const AsmToken &NextToken)
const;
1691 bool isRegOrOperandModifier(
const AsmToken &Token,
1692 const AsmToken &NextToken)
const;
1693 bool isNamedOperandModifier(
const AsmToken &Token,
1694 const AsmToken &NextToken)
const;
1695 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1696 const AsmToken &NextToken)
const;
1697 bool parseSP3NegModifier();
1704 bool AllowImm =
true);
1706 bool AllowImm =
true);
1711 AMDGPUOperand::ImmTy ImmTy);
1716 AMDGPUOperand::ImmTy
Type);
1720 AMDGPUOperand::ImmTy
Type);
1724 AMDGPUOperand::ImmTy
Type);
1728 ParseStatus parseDfmtNfmt(int64_t &
Format);
1729 ParseStatus parseUfmt(int64_t &
Format);
1730 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1732 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1735 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1736 ParseStatus parseNumericFormat(int64_t &
Format);
1740 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1741 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1746 bool parseCnt(int64_t &IntVal);
1749 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1750 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1753 bool parseDelay(int64_t &Delay);
1759 struct OperandInfoTy {
1762 bool IsSymbolic =
false;
1763 bool IsDefined =
false;
1765 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1768 struct StructuredOpField : OperandInfoTy {
1772 bool IsDefined =
false;
1774 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1775 unsigned Width, int64_t
Default)
1776 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1777 virtual ~StructuredOpField() =
default;
1779 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1780 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1784 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1786 return Error(Parser,
"not supported on this GPU");
1788 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1796 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1797 OperandInfoTy &Stream);
1798 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1799 const OperandInfoTy &Stream);
1801 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1802 OperandInfoTy &Width);
1807 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1814 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1816 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1820 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1826 bool validateConstantBusLimitations(
const MCInst &Inst,
1828 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1831 bool tryVOPD(
const MCInst &Inst);
1832 bool tryVOPD3(
const MCInst &Inst);
1833 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1835 bool validateIntClampSupported(
const MCInst &Inst);
1836 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1837 bool validateMIMGGatherDMask(
const MCInst &Inst);
1839 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1841 bool validateMIMGD16(
const MCInst &Inst);
1843 bool validateTensorR128(
const MCInst &Inst);
1844 bool validateMIMGMSAA(
const MCInst &Inst);
1845 bool validateOpSel(
const MCInst &Inst);
1846 bool validateTrue16OpSel(
const MCInst &Inst);
1847 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1849 bool validateVccOperand(MCRegister
Reg)
const;
1854 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1855 bool validateVGPRAlign(
const MCInst &Inst)
const;
1859 bool validateDivScale(
const MCInst &Inst);
1864 const unsigned CPol);
1868 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1869 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1870 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1871 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1874 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1875 ArrayRef<unsigned> Variants);
1876 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1878 bool isId(
const StringRef Id)
const;
1879 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1881 StringRef getId()
const;
1882 bool trySkipId(
const StringRef Id);
1883 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1887 bool parseString(StringRef &Val,
1888 const StringRef ErrMsg =
"expected a string");
1889 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1895 StringRef getTokenStr()
const;
1896 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1898 SMLoc getLoc()
const;
1902 void onBeginOfFile()
override;
1906 void emitTargetDirective();
1907 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1919 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1920 const unsigned MaxVal,
const Twine &ErrMsg,
1922 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1923 const unsigned MinVal,
const unsigned MaxVal,
1924 const StringRef ErrMsg);
1926 bool parseSwizzleOffset(int64_t &
Imm);
1927 bool parseSwizzleMacro(int64_t &
Imm);
1928 bool parseSwizzleQuadPerm(int64_t &
Imm);
1929 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1930 bool parseSwizzleBroadcast(int64_t &
Imm);
1931 bool parseSwizzleSwap(int64_t &
Imm);
1932 bool parseSwizzleReverse(int64_t &
Imm);
1933 bool parseSwizzleFFT(int64_t &
Imm);
1934 bool parseSwizzleRotate(int64_t &
Imm);
1937 int64_t parseGPRIdxMacro();
1940 cvtMubufImpl(Inst,
Operands,
false);
1943 cvtMubufImpl(Inst,
Operands,
true);
1949 OptionalImmIndexMap &OptionalIdx);
1958 OptionalImmIndexMap &OptionalIdx);
1960 OptionalImmIndexMap &OptionalIdx);
1964 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1966 bool parseDimId(
unsigned &Encoding);
1968 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1972 int64_t parseDPPCtrlSel(StringRef Ctrl);
1973 int64_t parseDPPCtrlPerm();
1979 bool IsDPP8 =
false);
1985 AMDGPUOperand::ImmTy
Type);
1993 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1996 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1997 bool SkipSrcVcc =
false);
2106bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2116 if (!isImmTy(ImmTyNone)) {
2121 if (getModifiers().
Lit != LitModifier::None)
2131 if (type == MVT::f64 || type == MVT::i64) {
2133 AsmParser->hasInv2PiInlineImm());
2136 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2155 APFloat::rmNearestTiesToEven, &Lost);
2162 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2164 AsmParser->hasInv2PiInlineImm());
2169 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2170 AsmParser->hasInv2PiInlineImm());
2174 if (type == MVT::f64 || type == MVT::i64) {
2176 AsmParser->hasInv2PiInlineImm());
2185 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2186 AsmParser->hasInv2PiInlineImm());
2190 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2191 AsmParser->hasInv2PiInlineImm());
2194bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2196 if (!isImmTy(ImmTyNone)) {
2201 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2206 if (type == MVT::f64 && hasFPModifiers()) {
2226 if (type == MVT::f64) {
2231 if (type == MVT::i64) {
2244 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2245 : (type == MVT::v2i16) ? MVT::f32
2246 : (type == MVT::v2f32) ? MVT::f32
2249 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2253bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2254 return isRegKind() &&
2255 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2258bool AMDGPUOperand::isVRegWithInputMods()
const {
2259 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2261 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2262 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2265template <
bool IsFake16>
2266bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2267 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2268 : AMDGPU::VGPR_16_Lo128RegClassID);
2271template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2272 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2273 : AMDGPU::VGPR_16RegClassID);
2276bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2277 if (AsmParser->isVI())
2279 if (AsmParser->isGFX9Plus())
2280 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2284bool AMDGPUOperand::isSDWAFP16Operand()
const {
2285 return isSDWAOperand(MVT::f16);
2288bool AMDGPUOperand::isSDWAFP32Operand()
const {
2289 return isSDWAOperand(MVT::f32);
2292bool AMDGPUOperand::isSDWAInt16Operand()
const {
2293 return isSDWAOperand(MVT::i16);
2296bool AMDGPUOperand::isSDWAInt32Operand()
const {
2297 return isSDWAOperand(MVT::i32);
2300bool AMDGPUOperand::isBoolReg()
const {
2301 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2302 (AsmParser->isWave32() && isSCSrc_b32()));
2306 unsigned Size)
const {
2307 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2322void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2323 bool ApplyModifiers)
const {
2333 addLiteralImmOperand(Inst,
Imm.Val,
2334 ApplyModifiers & isImmTy(ImmTyNone) &&
2335 Imm.Mods.hasFPModifiers());
2337 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2342void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2343 bool ApplyModifiers)
const {
2344 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2349 if (ApplyModifiers) {
2351 const unsigned Size =
2353 Val = applyInputFPModifiers(Val,
Size);
2357 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2359 bool CanUse64BitLiterals =
2362 MCContext &Ctx = AsmParser->getContext();
2373 if (
Lit == LitModifier::None &&
2375 AsmParser->hasInv2PiInlineImm())) {
2383 bool HasMandatoryLiteral =
2386 if (
Literal.getLoBits(32) != 0 &&
2387 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2388 !HasMandatoryLiteral) {
2389 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2391 "Can't encode literal as exact 64-bit floating-point operand. "
2392 "Low 32-bits will be set to zero");
2393 Val &= 0xffffffff00000000u;
2399 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2405 Lit = LitModifier::Lit64;
2406 }
else if (
Lit == LitModifier::Lit) {
2420 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2422 Lit = LitModifier::Lit64;
2429 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2430 Literal == 0x3fc45f306725feed) {
2469 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2476 if (
Lit != LitModifier::None) {
2507 if (
Lit == LitModifier::None &&
2517 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2525 if (
Lit == LitModifier::None &&
2533 if (!AsmParser->has64BitLiterals()) {
2534 Val =
static_cast<uint64_t>(Val) << 32;
2541 if (
Lit == LitModifier::Lit ||
2543 Val =
static_cast<uint64_t>(Val) << 32;
2547 if (
Lit == LitModifier::Lit)
2573 if (
Lit != LitModifier::None) {
2581void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2587bool AMDGPUOperand::isInlineValue()
const {
2595void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2606 if (Is == IS_VGPR) {
2611 return AMDGPU::VGPR_32RegClassID;
2613 return AMDGPU::VReg_64RegClassID;
2615 return AMDGPU::VReg_96RegClassID;
2617 return AMDGPU::VReg_128RegClassID;
2619 return AMDGPU::VReg_160RegClassID;
2621 return AMDGPU::VReg_192RegClassID;
2623 return AMDGPU::VReg_224RegClassID;
2625 return AMDGPU::VReg_256RegClassID;
2627 return AMDGPU::VReg_288RegClassID;
2629 return AMDGPU::VReg_320RegClassID;
2631 return AMDGPU::VReg_352RegClassID;
2633 return AMDGPU::VReg_384RegClassID;
2635 return AMDGPU::VReg_512RegClassID;
2637 return AMDGPU::VReg_1024RegClassID;
2639 }
else if (Is == IS_TTMP) {
2644 return AMDGPU::TTMP_32RegClassID;
2646 return AMDGPU::TTMP_64RegClassID;
2648 return AMDGPU::TTMP_128RegClassID;
2650 return AMDGPU::TTMP_256RegClassID;
2652 return AMDGPU::TTMP_512RegClassID;
2654 }
else if (Is == IS_SGPR) {
2659 return AMDGPU::SGPR_32RegClassID;
2661 return AMDGPU::SGPR_64RegClassID;
2663 return AMDGPU::SGPR_96RegClassID;
2665 return AMDGPU::SGPR_128RegClassID;
2667 return AMDGPU::SGPR_160RegClassID;
2669 return AMDGPU::SGPR_192RegClassID;
2671 return AMDGPU::SGPR_224RegClassID;
2673 return AMDGPU::SGPR_256RegClassID;
2675 return AMDGPU::SGPR_288RegClassID;
2677 return AMDGPU::SGPR_320RegClassID;
2679 return AMDGPU::SGPR_352RegClassID;
2681 return AMDGPU::SGPR_384RegClassID;
2683 return AMDGPU::SGPR_512RegClassID;
2685 }
else if (Is == IS_AGPR) {
2690 return AMDGPU::AGPR_32RegClassID;
2692 return AMDGPU::AReg_64RegClassID;
2694 return AMDGPU::AReg_96RegClassID;
2696 return AMDGPU::AReg_128RegClassID;
2698 return AMDGPU::AReg_160RegClassID;
2700 return AMDGPU::AReg_192RegClassID;
2702 return AMDGPU::AReg_224RegClassID;
2704 return AMDGPU::AReg_256RegClassID;
2706 return AMDGPU::AReg_288RegClassID;
2708 return AMDGPU::AReg_320RegClassID;
2710 return AMDGPU::AReg_352RegClassID;
2712 return AMDGPU::AReg_384RegClassID;
2714 return AMDGPU::AReg_512RegClassID;
2716 return AMDGPU::AReg_1024RegClassID;
2724 .
Case(
"exec", AMDGPU::EXEC)
2725 .
Case(
"vcc", AMDGPU::VCC)
2726 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2727 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2728 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2730 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2732 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2734 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2736 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2737 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2738 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2740 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2742 .
Case(
"m0", AMDGPU::M0)
2743 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2745 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2747 .
Case(
"scc", AMDGPU::SRC_SCC)
2748 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2749 .
Case(
"tba", AMDGPU::TBA)
2750 .
Case(
"tma", AMDGPU::TMA)
2751 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2752 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2753 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2754 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2755 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2756 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2757 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2758 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2759 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2760 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2761 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2762 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2763 .
Case(
"pc", AMDGPU::PC_REG)
2764 .
Case(
"null", AMDGPU::SGPR_NULL)
2768bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2769 SMLoc &EndLoc,
bool RestoreOnFailure) {
2770 auto R = parseRegister();
2774 RegNo =
R->getReg();
2775 StartLoc =
R->getStartLoc();
2776 EndLoc =
R->getEndLoc();
2780bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2782 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2785ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2787 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2788 bool PendingErrors = getParser().hasPendingError();
2789 getParser().clearPendingErrors();
2797bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2798 RegisterKind RegKind,
2800 RegisterKind RegKind1, SMLoc Loc) {
2802 if (RegKind == IS_SGPR) {
2803 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2804 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2805 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2811 if (RegKind != RegKind1) {
2812 Error(Loc,
"registers in a list must be of the same kind");
2818 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2823 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2824 Reg = AMDGPU::FLAT_SCR;
2828 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2829 Reg = AMDGPU::XNACK_MASK;
2833 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2838 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2843 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2848 Error(Loc,
"register does not fit in the list");
2854 if (Reg1 !=
Reg + RegWidth / 32) {
2855 Error(Loc,
"registers in a list must have consecutive indices");
2871 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2872 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2876 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2882 if (Str.starts_with(
Reg.Name))
2888 return !Str.getAsInteger(10, Num);
2891bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2892 const AsmToken &NextToken)
const {
2907 StringRef RegSuffix = Str.substr(
RegName.size());
2908 if (!RegSuffix.
empty()) {
2925bool AMDGPUAsmParser::isRegister() {
2926 return isRegister(
getToken(), peekToken());
2929MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2930 unsigned SubReg,
unsigned RegWidth,
2934 unsigned AlignSize = 1;
2935 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2941 if (RegNum % AlignSize != 0) {
2942 Error(Loc,
"invalid register alignment");
2943 return MCRegister();
2946 unsigned RegIdx = RegNum / AlignSize;
2949 Error(Loc,
"invalid or unsupported register size");
2950 return MCRegister();
2954 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2955 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2956 Error(Loc,
"register index is out of range");
2957 return AMDGPU::NoRegister;
2960 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2961 Error(Loc,
"register index is out of range");
2962 return MCRegister();
2978bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2980 int64_t RegLo, RegHi;
2984 SMLoc FirstIdxLoc = getLoc();
2991 SecondIdxLoc = getLoc();
3002 Error(FirstIdxLoc,
"invalid register index");
3007 Error(SecondIdxLoc,
"invalid register index");
3011 if (RegLo > RegHi) {
3012 Error(FirstIdxLoc,
"first register index should not exceed second index");
3016 if (RegHi == RegLo) {
3017 StringRef RegSuffix = getTokenStr();
3018 if (RegSuffix ==
".l") {
3019 SubReg = AMDGPU::lo16;
3021 }
else if (RegSuffix ==
".h") {
3022 SubReg = AMDGPU::hi16;
3027 Num =
static_cast<unsigned>(RegLo);
3028 RegWidth = 32 * ((RegHi - RegLo) + 1);
3033MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3036 SmallVectorImpl<AsmToken> &Tokens) {
3042 RegKind = IS_SPECIAL;
3049MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3052 SmallVectorImpl<AsmToken> &Tokens) {
3054 StringRef
RegName = getTokenStr();
3055 auto Loc = getLoc();
3059 Error(Loc,
"invalid register name");
3060 return MCRegister();
3068 unsigned SubReg = NoSubRegister;
3069 bool IsRange =
false;
3070 if (!RegSuffix.
empty()) {
3072 SubReg = AMDGPU::lo16;
3074 SubReg = AMDGPU::hi16;
3078 Error(Loc,
"invalid register index");
3079 return MCRegister();
3085 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3086 return MCRegister();
3090 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3091 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3092 if (RegKind == IS_SGPR && IsRange
3093 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3094 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3095 Error(Loc,
"register index is out of range");
3096 return MCRegister();
3102MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3103 unsigned &RegNum,
unsigned &RegWidth,
3104 SmallVectorImpl<AsmToken> &Tokens) {
3106 auto ListLoc = getLoc();
3109 "expected a register or a list of registers")) {
3110 return MCRegister();
3115 auto Loc = getLoc();
3116 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3117 return MCRegister();
3118 if (RegWidth != 32) {
3119 Error(Loc,
"expected a single 32-bit register");
3120 return MCRegister();
3124 RegisterKind NextRegKind;
3126 unsigned NextRegNum, NextRegWidth;
3129 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3131 return MCRegister();
3133 if (NextRegWidth != 32) {
3134 Error(Loc,
"expected a single 32-bit register");
3135 return MCRegister();
3137 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3139 return MCRegister();
3143 "expected a comma or a closing square bracket")) {
3144 return MCRegister();
3148 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3153bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3154 MCRegister &
Reg,
unsigned &RegNum,
3156 SmallVectorImpl<AsmToken> &Tokens) {
3157 auto Loc = getLoc();
3161 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3163 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3165 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3170 assert(Parser.hasPendingError());
3174 if (!subtargetHasRegister(*
TRI,
Reg)) {
3175 if (
Reg == AMDGPU::SGPR_NULL) {
3176 Error(Loc,
"'null' operand is not supported on this GPU");
3179 " register not available on this GPU");
3187bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3188 MCRegister &
Reg,
unsigned &RegNum,
3190 bool RestoreOnFailure ) {
3194 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3195 if (RestoreOnFailure) {
3196 while (!Tokens.
empty()) {
3205std::optional<StringRef>
3206AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3209 return StringRef(
".amdgcn.next_free_vgpr");
3211 return StringRef(
".amdgcn.next_free_sgpr");
3213 return std::nullopt;
3217void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3218 auto SymbolName = getGprCountSymbolName(RegKind);
3219 assert(SymbolName &&
"initializing invalid register kind");
3225bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3226 unsigned DwordRegIndex,
3227 unsigned RegWidth) {
3232 auto SymbolName = getGprCountSymbolName(RegKind);
3237 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3241 return !
Error(getLoc(),
3242 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3246 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3248 if (OldCount <= NewMax)
3254std::unique_ptr<AMDGPUOperand>
3255AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3257 SMLoc StartLoc = Tok.getLoc();
3258 SMLoc EndLoc = Tok.getEndLoc();
3259 RegisterKind RegKind;
3261 unsigned RegNum, RegWidth;
3263 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3267 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3270 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3271 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3278 if (isRegister() || isModifier())
3281 if (
Lit == LitModifier::None) {
3282 if (trySkipId(
"lit"))
3283 Lit = LitModifier::Lit;
3284 else if (trySkipId(
"lit64"))
3285 Lit = LitModifier::Lit64;
3287 if (
Lit != LitModifier::None) {
3290 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3299 const auto &NextTok = peekToken();
3302 bool Negate =
false;
3310 AMDGPUOperand::Modifiers Mods;
3318 StringRef Num = getTokenStr();
3321 APFloat RealVal(APFloat::IEEEdouble());
3322 auto roundMode = APFloat::rmNearestTiesToEven;
3323 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3326 RealVal.changeSign();
3329 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3330 S, AMDGPUOperand::ImmTyNone,
true));
3331 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3332 Op.setModifiers(Mods);
3341 if (HasSP3AbsModifier) {
3350 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3353 if (Parser.parseExpression(Expr))
3357 if (Expr->evaluateAsAbsolute(IntVal)) {
3359 return Error(S,
"literal value out of range");
3360 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3361 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3362 Op.setModifiers(Mods);
3364 if (
Lit != LitModifier::None)
3366 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3379 if (
auto R = parseRegister()) {
3389 ParseStatus Res = parseReg(
Operands);
3397bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3398 const AsmToken &NextToken)
const {
3401 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3406bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3407 const AsmToken &NextToken)
const {
3411bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3412 const AsmToken &NextToken)
const {
3413 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3416bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3417 const AsmToken &NextToken)
const {
3418 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3434bool AMDGPUAsmParser::isModifier() {
3437 AsmToken NextToken[2];
3438 peekTokens(NextToken);
3440 return isOperandModifier(Tok, NextToken[0]) ||
3442 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3443 isOpcodeModifierWithVal(Tok, NextToken[0]);
3468bool AMDGPUAsmParser::parseSP3NegModifier() {
3470 AsmToken NextToken[2];
3471 peekTokens(NextToken);
3474 (isRegister(NextToken[0], NextToken[1]) ||
3492 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3494 SP3Neg = parseSP3NegModifier();
3497 Neg = trySkipId(
"neg");
3499 return Error(Loc,
"expected register or immediate");
3503 Abs = trySkipId(
"abs");
3508 if (trySkipId(
"lit")) {
3509 Lit = LitModifier::Lit;
3512 }
else if (trySkipId(
"lit64")) {
3513 Lit = LitModifier::Lit64;
3516 if (!has64BitLiterals())
3517 return Error(Loc,
"lit64 is not supported on this GPU");
3523 return Error(Loc,
"expected register or immediate");
3532 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3536 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3537 Error(Loc,
"expected immediate with lit modifier");
3539 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3545 if (
Lit != LitModifier::None &&
3549 AMDGPUOperand::Modifiers Mods;
3550 Mods.Abs = Abs || SP3Abs;
3551 Mods.Neg = Neg || SP3Neg;
3554 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3555 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3557 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3558 Op.setModifiers(Mods);
3566 bool Sext = trySkipId(
"sext");
3567 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3582 AMDGPUOperand::Modifiers Mods;
3585 if (Mods.hasIntModifiers()) {
3586 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3588 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3589 Op.setModifiers(Mods);
3596 return parseRegOrImmWithFPInputMods(
Operands,
false);
3600 return parseRegOrImmWithIntInputMods(
Operands,
false);
3604 auto Loc = getLoc();
3605 if (trySkipId(
"off")) {
3607 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3614 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3623unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3628 return Match_InvalidOperand;
3630 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3631 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3634 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3636 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3637 return Match_InvalidOperand;
3645 if (tryAnotherVOPDEncoding(Inst))
3646 return Match_InvalidOperand;
3648 return Match_Success;
3652 static const unsigned Variants[] = {
3661ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3662 if (isForcedDPP() && isForcedVOP3()) {
3666 if (getForcedEncodingSize() == 32) {
3671 if (isForcedVOP3()) {
3676 if (isForcedSDWA()) {
3682 if (isForcedDPP()) {
3690StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3691 if (isForcedDPP() && isForcedVOP3())
3694 if (getForcedEncodingSize() == 32)
3710AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3714 case AMDGPU::FLAT_SCR:
3716 case AMDGPU::VCC_LO:
3717 case AMDGPU::VCC_HI:
3724 return MCRegister();
3731bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3732 unsigned OpIdx)
const {
3740 const MCOperand &MO = Inst.
getOperand(OpIdx);
3789unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3795 case AMDGPU::V_LSHLREV_B64_e64:
3796 case AMDGPU::V_LSHLREV_B64_gfx10:
3797 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3798 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3799 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3800 case AMDGPU::V_LSHRREV_B64_e64:
3801 case AMDGPU::V_LSHRREV_B64_gfx10:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3803 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3804 case AMDGPU::V_ASHRREV_I64_e64:
3805 case AMDGPU::V_ASHRREV_I64_gfx10:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3807 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3808 case AMDGPU::V_LSHL_B64_e64:
3809 case AMDGPU::V_LSHR_B64_e64:
3810 case AMDGPU::V_ASHR_I64_e64:
3823 bool AddMandatoryLiterals =
false) {
3826 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3830 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3832 return {getNamedOperandIdx(Opcode, OpName::src0X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3834 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3835 getNamedOperandIdx(Opcode, OpName::src0Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3837 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3842 return {getNamedOperandIdx(Opcode, OpName::src0),
3843 getNamedOperandIdx(Opcode, OpName::src1),
3844 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3847bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3848 const MCOperand &MO = Inst.
getOperand(OpIdx);
3850 return !isInlineConstant(Inst, OpIdx);
3857 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3868 const unsigned Opcode = Inst.
getOpcode();
3869 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3872 if (!LaneSelOp.
isReg())
3875 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3878bool AMDGPUAsmParser::validateConstantBusLimitations(
3880 const unsigned Opcode = Inst.
getOpcode();
3881 const MCInstrDesc &
Desc = MII.
get(Opcode);
3882 MCRegister LastSGPR;
3883 unsigned ConstantBusUseCount = 0;
3884 unsigned NumLiterals = 0;
3885 unsigned LiteralSize;
3901 SmallDenseSet<MCRegister> SGPRsUsed;
3902 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3904 SGPRsUsed.
insert(SGPRUsed);
3905 ++ConstantBusUseCount;
3910 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3912 for (
int OpIdx : OpIndices) {
3916 const MCOperand &MO = Inst.
getOperand(OpIdx);
3917 if (usesConstantBus(Inst, OpIdx)) {
3926 if (SGPRsUsed.
insert(LastSGPR).second) {
3927 ++ConstantBusUseCount;
3947 if (NumLiterals == 0) {
3950 }
else if (LiteralSize !=
Size) {
3956 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3958 "invalid operand (violates constant bus restrictions)");
3965std::optional<unsigned>
3966AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3968 const unsigned Opcode = Inst.
getOpcode();
3974 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3975 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3989 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3993 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3994 int I = getNamedOperandIdx(Opcode, OpName);
3998 int64_t
Imm =
Op.getImm();
4004 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4005 OpName::vsrc2Y, OpName::imm}) {
4006 int I = getNamedOperandIdx(Opcode, OpName);
4016 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4017 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4019 return InvalidCompOprIdx;
4022bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4029 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4030 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4031 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4033 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4037 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4038 if (!InvalidCompOprIdx.has_value())
4041 auto CompOprIdx = *InvalidCompOprIdx;
4044 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4045 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4048 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4049 if (CompOprIdx == VOPD::Component::DST) {
4051 Error(Loc,
"dst registers must be distinct");
4053 Error(Loc,
"one dst register must be even and the other odd");
4055 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4056 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4057 " operands must use different VGPR banks");
4065bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4067 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4068 if (!InvalidCompOprIdx.has_value())
4072 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4073 if (InvalidCompOprIdx.has_value()) {
4078 if (*InvalidCompOprIdx == VOPD::Component::DST)
4091bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4092 const unsigned Opcode = Inst.
getOpcode();
4107 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4108 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4109 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4110 int I = getNamedOperandIdx(Opcode, OpName);
4117 return !tryVOPD3(Inst);
4122bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4127 return tryVOPD(Inst);
4128 return tryVOPD3(Inst);
4131bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4136 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4144bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4152 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4153 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4154 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4159 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4160 hasBVHRayTracingInsts())
4163 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4164 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4169 bool IsPackedD16 =
false;
4172 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4173 IsPackedD16 = D16Idx >= 0;
4175 DataSize = (DataSize + 1) / 2;
4178 if ((VDataSize / 4) == DataSize + TFESize)
4181 StringRef Modifiers;
4183 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4185 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4187 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4191bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4200 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4202 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4203 AMDGPU::OpName RSrcOpName =
4205 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4206 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4207 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4211 assert(SrsrcIdx > VAddr0Idx);
4214 if (BaseOpcode->
BVH) {
4215 if (IsA16 == BaseOpcode->
A16)
4217 Error(IDLoc,
"image address size does not match a16");
4223 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4224 unsigned ActualAddrSize =
4225 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4227 unsigned ExpectedAddrSize =
4231 if (hasPartialNSAEncoding() &&
4233 int VAddrLastIdx = SrsrcIdx - 1;
4234 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4236 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4239 if (ExpectedAddrSize > 12)
4240 ExpectedAddrSize = 16;
4245 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4249 if (ActualAddrSize == ExpectedAddrSize)
4252 Error(IDLoc,
"image address size does not match dim and a16");
4256bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4263 if (!
Desc.mayLoad() || !
Desc.mayStore())
4266 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4273 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4276bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4283 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4291 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4294bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4308 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4309 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4316bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4323 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4326 if (!BaseOpcode->
MSAA)
4329 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4335 return DimInfo->
MSAA;
4340 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4342 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4352bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4360 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4363 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4371 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4375bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4380 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4383 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4386 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4394 "source operand must be either a VGPR or an inline constant");
4401bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4406 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4409 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4413 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4415 "inline constants are not allowed for this operand");
4422bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4430 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4431 if (BlgpIdx != -1) {
4432 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4433 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4443 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4445 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4450 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4452 "wrong register tuple size for blgp value " + Twine(BLGP));
4460 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4464 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4468 MCRegister Src2Reg = Src2.
getReg();
4470 if (Src2Reg == DstReg)
4475 .getSizeInBits() <= 128)
4478 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4480 "source 2 operand must not partially overlap with dst");
4487bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4491 case V_DIV_SCALE_F32_gfx6_gfx7:
4492 case V_DIV_SCALE_F32_vi:
4493 case V_DIV_SCALE_F32_gfx10:
4494 case V_DIV_SCALE_F64_gfx6_gfx7:
4495 case V_DIV_SCALE_F64_vi:
4496 case V_DIV_SCALE_F64_gfx10:
4503 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4504 AMDGPU::OpName::src2_modifiers}) {
4515bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4522 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4531bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4537 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4544 case AMDGPU::V_SUBREV_F32_e32:
4545 case AMDGPU::V_SUBREV_F32_e64:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4547 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4548 case AMDGPU::V_SUBREV_F32_e32_vi:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4550 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4551 case AMDGPU::V_SUBREV_F32_e64_vi:
4553 case AMDGPU::V_SUBREV_CO_U32_e32:
4554 case AMDGPU::V_SUBREV_CO_U32_e64:
4555 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4556 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4558 case AMDGPU::V_SUBBREV_U32_e32:
4559 case AMDGPU::V_SUBBREV_U32_e64:
4560 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4561 case AMDGPU::V_SUBBREV_U32_e32_vi:
4562 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4563 case AMDGPU::V_SUBBREV_U32_e64_vi:
4565 case AMDGPU::V_SUBREV_U32_e32:
4566 case AMDGPU::V_SUBREV_U32_e64:
4567 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4568 case AMDGPU::V_SUBREV_U32_e32_vi:
4569 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4570 case AMDGPU::V_SUBREV_U32_e64_vi:
4572 case AMDGPU::V_SUBREV_F16_e32:
4573 case AMDGPU::V_SUBREV_F16_e64:
4574 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4575 case AMDGPU::V_SUBREV_F16_e32_vi:
4576 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4577 case AMDGPU::V_SUBREV_F16_e64_vi:
4579 case AMDGPU::V_SUBREV_U16_e32:
4580 case AMDGPU::V_SUBREV_U16_e64:
4581 case AMDGPU::V_SUBREV_U16_e32_vi:
4582 case AMDGPU::V_SUBREV_U16_e64_vi:
4584 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4586 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4589 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4591 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4592 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4595 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4597 case AMDGPU::V_LSHRREV_B32_e32:
4598 case AMDGPU::V_LSHRREV_B32_e64:
4599 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4601 case AMDGPU::V_LSHRREV_B32_e32_vi:
4602 case AMDGPU::V_LSHRREV_B32_e64_vi:
4603 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4604 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4606 case AMDGPU::V_ASHRREV_I32_e32:
4607 case AMDGPU::V_ASHRREV_I32_e64:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4609 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4610 case AMDGPU::V_ASHRREV_I32_e32_vi:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4612 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4613 case AMDGPU::V_ASHRREV_I32_e64_vi:
4615 case AMDGPU::V_LSHLREV_B32_e32:
4616 case AMDGPU::V_LSHLREV_B32_e64:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4618 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4619 case AMDGPU::V_LSHLREV_B32_e32_vi:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4621 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4622 case AMDGPU::V_LSHLREV_B32_e64_vi:
4624 case AMDGPU::V_LSHLREV_B16_e32:
4625 case AMDGPU::V_LSHLREV_B16_e64:
4626 case AMDGPU::V_LSHLREV_B16_e32_vi:
4627 case AMDGPU::V_LSHLREV_B16_e64_vi:
4628 case AMDGPU::V_LSHLREV_B16_gfx10:
4630 case AMDGPU::V_LSHRREV_B16_e32:
4631 case AMDGPU::V_LSHRREV_B16_e64:
4632 case AMDGPU::V_LSHRREV_B16_e32_vi:
4633 case AMDGPU::V_LSHRREV_B16_e64_vi:
4634 case AMDGPU::V_LSHRREV_B16_gfx10:
4636 case AMDGPU::V_ASHRREV_I16_e32:
4637 case AMDGPU::V_ASHRREV_I16_e64:
4638 case AMDGPU::V_ASHRREV_I16_e32_vi:
4639 case AMDGPU::V_ASHRREV_I16_e64_vi:
4640 case AMDGPU::V_ASHRREV_I16_gfx10:
4642 case AMDGPU::V_LSHLREV_B64_e64:
4643 case AMDGPU::V_LSHLREV_B64_gfx10:
4644 case AMDGPU::V_LSHLREV_B64_vi:
4646 case AMDGPU::V_LSHRREV_B64_e64:
4647 case AMDGPU::V_LSHRREV_B64_gfx10:
4648 case AMDGPU::V_LSHRREV_B64_vi:
4650 case AMDGPU::V_ASHRREV_I64_e64:
4651 case AMDGPU::V_ASHRREV_I64_gfx10:
4652 case AMDGPU::V_ASHRREV_I64_vi:
4654 case AMDGPU::V_PK_LSHLREV_B16:
4655 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4656 case AMDGPU::V_PK_LSHLREV_B16_vi:
4658 case AMDGPU::V_PK_LSHRREV_B16:
4659 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4660 case AMDGPU::V_PK_LSHRREV_B16_vi:
4661 case AMDGPU::V_PK_ASHRREV_I16:
4662 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4663 case AMDGPU::V_PK_ASHRREV_I16_vi:
4670bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4672 const unsigned Opcode = Inst.
getOpcode();
4681 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4682 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4686 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4690 "lds_direct is not supported on this GPU");
4696 "lds_direct cannot be used with this instruction");
4700 if (SrcName != OpName::src0) {
4702 "lds_direct may be used as src0 only");
4712 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4713 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4714 if (
Op.isFlatOffset())
4715 return Op.getStartLoc();
4720bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4723 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4728 return validateFlatOffset(Inst,
Operands);
4731 return validateSMEMOffset(Inst,
Operands);
4736 const unsigned OffsetSize = 24;
4737 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4739 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4740 "-bit unsigned offset for buffer ops");
4744 const unsigned OffsetSize = 16;
4745 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4747 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4754bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4760 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4764 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4766 "flat offset modifier is not supported on this GPU");
4773 bool AllowNegative =
4775 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4777 Twine(
"expected a ") +
4778 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4779 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4788 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4789 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4790 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4791 return Op.getStartLoc();
4796bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4805 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4821 ?
"expected a 23-bit unsigned offset for buffer ops"
4822 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4823 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4824 :
"expected a 21-bit signed offset");
4829bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4832 const MCInstrDesc &
Desc = MII.
get(Opcode);
4836 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4837 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4839 const int OpIndices[] = {Src0Idx, Src1Idx};
4841 unsigned NumExprs = 0;
4842 unsigned NumLiterals = 0;
4845 for (
int OpIdx : OpIndices) {
4849 const MCOperand &MO = Inst.
getOperand(OpIdx);
4853 std::optional<int64_t>
Imm;
4856 }
else if (MO.
isExpr()) {
4865 if (!
Imm.has_value()) {
4867 }
else if (!isInlineConstant(Inst, OpIdx)) {
4871 if (NumLiterals == 0 || LiteralValue !=
Value) {
4879 if (NumLiterals + NumExprs <= 1)
4883 "only one unique literal operand is allowed");
4887bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4890 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4899 if (OpSelIdx != -1) {
4903 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4904 if (OpSelHiIdx != -1) {
4913 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4923 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4924 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4925 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4926 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4928 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4929 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4935 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4937 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4947 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4948 if (Src2Idx != -1) {
4949 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4959bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4960 if (!hasTrue16Insts())
4962 const MCRegisterInfo *MRI = getMRI();
4964 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4970 if (OpSelOpValue == 0)
4972 unsigned OpCount = 0;
4973 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4974 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4975 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4982 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4983 if (OpSelOpIsHi != VGPRSuffixIsHi)
4992bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4993 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5005 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5016 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5017 AMDGPU::OpName::src1_modifiers,
5018 AMDGPU::OpName::src2_modifiers};
5020 for (
unsigned i = 0; i < 3; ++i) {
5030bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5033 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5034 if (DppCtrlIdx >= 0) {
5041 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5042 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5043 :
"DP ALU dpp only supports row_newbcast");
5048 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5049 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5052 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5054 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5058 "invalid operand for instruction");
5063 "src1 immediate operand invalid for instruction");
5073bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5074 return (
Reg == AMDGPU::VCC && isWave64()) ||
5075 (
Reg == AMDGPU::VCC_LO && isWave32());
5079bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5082 const MCInstrDesc &
Desc = MII.
get(Opcode);
5083 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5090 std::optional<unsigned> LiteralOpIdx;
5093 for (
int OpIdx : OpIndices) {
5097 const MCOperand &MO = Inst.
getOperand(OpIdx);
5103 std::optional<int64_t>
Imm;
5109 bool IsAnotherLiteral =
false;
5110 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5111 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5112 if (!
Imm.has_value()) {
5114 IsAnotherLiteral =
true;
5115 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5120 HasMandatoryLiteral);
5121 unsigned OpTy =
Desc.operands()[OpIdx].OperandType;
5131 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5132 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5134 "invalid operand for instruction");
5139 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5140 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5142 "invalid operand for instruction");
5146 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5153 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5154 !getFeatureBits()[FeatureVOP3Literal]) {
5156 "literal operands are not supported");
5160 if (LiteralOpIdx && IsAnotherLiteral) {
5162 getOperandLoc(
Operands, *LiteralOpIdx)),
5163 "only one unique literal operand is allowed");
5167 if (IsAnotherLiteral)
5168 LiteralOpIdx = OpIdx;
5177 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5191bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5197 ? AMDGPU::OpName::data0
5198 : AMDGPU::OpName::vdata;
5200 const MCRegisterInfo *MRI = getMRI();
5201 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5205 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5206 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5210 auto FB = getFeatureBits();
5211 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5212 if (DataAreg < 0 || DstAreg < 0)
5214 return DstAreg == DataAreg;
5217 return DstAreg < 1 && DataAreg < 1;
5220bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5221 auto FB = getFeatureBits();
5222 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5226 const MCRegisterInfo *MRI = getMRI();
5229 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5232 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5236 case AMDGPU::DS_LOAD_TR6_B96:
5237 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5242 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5246 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5247 if (VAddrIdx != -1) {
5250 if ((
Sub - AMDGPU::VGPR0) & 1)
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5256 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5261 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5262 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5282 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5283 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5285 return Op.getStartLoc();
5290bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5293 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5296 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5299 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5300 auto FB = getFeatureBits();
5301 bool UsesNeg =
false;
5302 if (FB[AMDGPU::FeatureGFX940Insts]) {
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5305 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5307 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5312 if (IsNeg == UsesNeg)
5315 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5316 :
"invalid modifier: neg is not supported");
5321bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5327 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5330 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5333 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5336 if (
Reg == AMDGPU::SGPR_NULL)
5339 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5343bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5348 return validateGWS(Inst,
Operands);
5353 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5358 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5359 Error(S,
"gds modifier is not supported on this GPU");
5367bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5369 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5373 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5374 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5377 const MCRegisterInfo *MRI = getMRI();
5378 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5380 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5383 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5385 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5392bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5396 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5404 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5407 Error(S,
"scale_offset is not supported on this GPU");
5410 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5413 Error(S,
"nv is not supported on this GPU");
5418 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5421 Error(S,
"scale_offset is not supported for this instruction");
5425 return validateTHAndScopeBits(Inst,
Operands, CPol);
5429 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5430 Error(S,
"cache policy is not supported for SMRD instructions");
5434 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5441 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5445 "scc modifier is not supported for this instruction on this GPU");
5456 :
"instruction must use glc");
5461 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5464 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5466 :
"instruction must not use glc");
5474bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5476 const unsigned CPol) {
5481 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5488 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5493 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5501 return PrintError(
"invalid th value for SMEM instruction");
5508 return PrintError(
"scope and th combination is not valid");
5514 return PrintError(
"invalid th value for atomic instructions");
5517 return PrintError(
"invalid th value for store instructions");
5520 return PrintError(
"invalid th value for load instructions");
5526bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5530 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5532 Error(Loc,
"TFE modifier has no meaning for store instructions");
5540bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5546 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5550 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5553 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5554 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5563 "wrong register tuple size for " +
5568 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5569 !validateFmt(BFmt, AMDGPU::OpName::src1))
5573 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5574 if (AScaleIdx == -1)
5578 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5582 "invalid matrix and scale format combination");
5589bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5591 if (!validateLdsDirect(Inst,
Operands))
5593 if (!validateTrue16OpSel(Inst)) {
5595 "op_sel operand conflicts with 16-bit operand suffix");
5598 if (!validateSOPLiteral(Inst,
Operands))
5600 if (!validateVOPLiteral(Inst,
Operands)) {
5603 if (!validateConstantBusLimitations(Inst,
Operands)) {
5606 if (!validateVOPD(Inst,
Operands)) {
5609 if (!validateIntClampSupported(Inst)) {
5611 "integer clamping is not supported on this GPU");
5614 if (!validateOpSel(Inst)) {
5616 "invalid op_sel operand");
5619 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5621 "invalid neg_lo operand");
5624 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5626 "invalid neg_hi operand");
5629 if (!validateDPP(Inst,
Operands)) {
5633 if (!validateMIMGD16(Inst)) {
5635 "d16 modifier is not supported on this GPU");
5638 if (!validateMIMGDim(Inst,
Operands)) {
5639 Error(IDLoc,
"missing dim operand");
5642 if (!validateTensorR128(Inst)) {
5644 "instruction must set modifier r128=0");
5647 if (!validateMIMGMSAA(Inst)) {
5649 "invalid dim; must be MSAA type");
5652 if (!validateMIMGDataSize(Inst, IDLoc)) {
5655 if (!validateMIMGAddrSize(Inst, IDLoc))
5657 if (!validateMIMGAtomicDMask(Inst)) {
5659 "invalid atomic image dmask");
5662 if (!validateMIMGGatherDMask(Inst)) {
5664 "invalid image_gather dmask: only one bit must be set");
5667 if (!validateMovrels(Inst,
Operands)) {
5670 if (!validateOffset(Inst,
Operands)) {
5673 if (!validateMAIAccWrite(Inst,
Operands)) {
5676 if (!validateMAISrc2(Inst,
Operands)) {
5679 if (!validateMFMA(Inst,
Operands)) {
5682 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5686 if (!validateAGPRLdSt(Inst)) {
5689 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5690 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5691 :
"invalid register class: agpr loads and stores not supported on "
5695 if (!validateVGPRAlign(Inst)) {
5696 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5703 if (!validateBLGP(Inst,
Operands)) {
5707 if (!validateDivScale(Inst)) {
5708 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5711 if (!validateWaitCnt(Inst,
Operands)) {
5714 if (!validateTFE(Inst,
Operands)) {
5717 if (!validateWMMA(Inst,
Operands)) {
5726 unsigned VariantID = 0);
5730 unsigned VariantID);
5732bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5737bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5738 const FeatureBitset &FBS,
5739 ArrayRef<unsigned> Variants) {
5740 for (
auto Variant : Variants) {
5748bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5750 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5753 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5758 getParser().clearPendingErrors();
5762 StringRef VariantName = getMatchedVariantName();
5763 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5764 return Error(IDLoc, Twine(VariantName,
5765 " variant of this instruction is not supported"));
5769 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5770 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5772 FeatureBitset FeaturesWS32 = getFeatureBits();
5773 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5774 .
flip(AMDGPU::FeatureWavefrontSize32);
5775 FeatureBitset AvailableFeaturesWS32 =
5776 ComputeAvailableFeatures(FeaturesWS32);
5778 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5779 return Error(IDLoc,
"instruction requires wavesize=32");
5783 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5784 return Error(IDLoc,
"instruction not supported on this GPU (" +
5785 getSTI().
getCPU() +
")" +
": " + Mnemo);
5790 return Error(IDLoc,
"invalid instruction" + Suggestion);
5796 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5797 if (
Op.isToken() && InvalidOprIdx > 1) {
5798 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5799 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5804bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5808 bool MatchingInlineAsm) {
5811 unsigned Result = Match_Success;
5812 for (
auto Variant : getMatchedVariants()) {
5815 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5819 if (R == Match_Success || R == Match_MissingFeature ||
5820 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5821 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5822 Result != Match_MissingFeature)) {
5826 if (R == Match_Success)
5830 if (Result == Match_Success) {
5831 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5834 emitTargetDirective();
5842 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5849 case Match_MissingFeature:
5853 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5855 case Match_InvalidOperand: {
5856 SMLoc ErrorLoc = IDLoc;
5857 if (ErrorInfo != ~0ULL) {
5858 if (ErrorInfo >=
Operands.size()) {
5859 return Error(IDLoc,
"too few operands for instruction");
5861 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5862 if (ErrorLoc == SMLoc())
5866 return Error(ErrorLoc,
"invalid VOPDY instruction");
5868 return Error(ErrorLoc,
"invalid operand for instruction");
5871 case Match_MnemonicFail:
5877bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5882 if (getParser().parseAbsoluteExpression(Tmp)) {
5885 Ret =
static_cast<uint32_t
>(Tmp);
5889bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5890 if (!getSTI().getTargetTriple().isAMDGCN())
5891 return TokError(
"directive only supported for amdgcn architecture");
5893 std::string TargetIDDirective;
5894 SMLoc TargetStart = getTok().getLoc();
5895 if (getParser().parseEscapedString(TargetIDDirective))
5898 std::optional<AMDGPU::TargetID> MaybeParsed =
5901 return getParser().Error(TargetStart,
5902 "malformed target id '" + TargetIDDirective +
"'");
5905 const Triple &
TT = getSTI().getTargetTriple();
5911 return getParser().Error(
5912 TargetStart,
"target id '" + TargetIDDirective +
5913 "' specifies a processor that is not valid for "
5915 TT.getArchName() +
"'");
5918 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5919 getTargetStreamer().getTargetID();
5922 const Triple &STITriple = getSTI().getTargetTriple();
5923 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5924 return getParser().Error(
5925 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5926 " is incompatible with " +
5927 Twine(CurrentTargetID->toString()));
5931 StringRef DirectiveProcessor =
5934 if (DirectiveISA != ISA) {
5935 return getParser().Error(TargetStart,
5936 ".amdgcn_target directive processor " +
5937 Twine(DirectiveProcessor) +
5938 " does not match the specified processor " +
5939 Twine(getSTI().
getCPU()));
5945 CurrentTargetID->getXnackSetting())) {
5947 ".amdgcn_target directive has conflicting xnack settings");
5951 CurrentTargetID->getSramEccSetting())) {
5953 ".amdgcn_target directive has conflicting sramecc settings");
5959 getTargetStreamer().getTargetID()->setXnackSetting(
5961 getTargetStreamer().getTargetID()->setSramEccSetting(
5967bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5971bool AMDGPUAsmParser::calculateGPRBlocks(
5972 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5973 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5974 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5975 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5976 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
5981 const MCExpr *
NumSGPRs = NextFreeSGPR;
5982 int64_t EvaluatedSGPRs;
5984 if (
ISA.Major >= 10)
5989 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
5990 !Features.
test(FeatureSGPRInitBug) &&
5991 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
5992 return OutOfRangeError(SGPRRange);
5994 const MCExpr *ExtraSGPRs =
5998 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
5999 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6000 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6001 return OutOfRangeError(SGPRRange);
6003 if (Features.
test(FeatureSGPRInitBug))
6010 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6011 unsigned Granule) ->
const MCExpr * {
6015 const MCExpr *AlignToGPR =
6017 const MCExpr *DivGPR =
6023 VGPRBlocks = GetNumGPRBlocks(
6032bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6033 if (!getSTI().getTargetTriple().isAMDGCN())
6034 return TokError(
"directive only supported for amdgcn architecture");
6037 return TokError(
"directive only supported for amdhsa OS");
6039 StringRef KernelName;
6040 if (getParser().parseIdentifier(KernelName))
6047 AMDGPU::MCKernelDescriptor KD =
6057 const MCExpr *NextFreeVGPR = ZeroExpr;
6059 const MCExpr *NamedBarCnt = ZeroExpr;
6064 const MCExpr *NextFreeSGPR = ZeroExpr;
6067 unsigned ImpliedUserSGPRCount = 0;
6071 std::optional<unsigned> ExplicitUserSGPRCount;
6072 const MCExpr *ReserveVCC = OneExpr;
6073 const MCExpr *ReserveFlatScr = OneExpr;
6074 std::optional<bool> EnableWavefrontSize32;
6081 SMRange IDRange = getTok().getLocRange();
6082 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6085 if (ID ==
".end_amdhsa_kernel")
6088 if (!Seen.
insert(ID).second)
6089 return TokError(
".amdhsa_ directives cannot be repeated");
6091 SMLoc ValStart = getLoc();
6092 const MCExpr *ExprVal;
6093 if (getParser().parseExpression(ExprVal))
6095 SMLoc ValEnd = getLoc();
6096 SMRange ValRange = SMRange(ValStart, ValEnd);
6100 bool EvaluatableExpr;
6101 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6103 return OutOfRangeError(ValRange);
6107#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6108 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6109 return OutOfRangeError(RANGE); \
6110 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6115#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6117 return Error(IDRange.Start, "directive should have resolvable expression", \
6120 if (ID ==
".amdhsa_group_segment_fixed_size") {
6123 return OutOfRangeError(ValRange);
6125 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6128 return OutOfRangeError(ValRange);
6130 }
else if (ID ==
".amdhsa_kernarg_size") {
6132 return OutOfRangeError(ValRange);
6134 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6136 ExplicitUserSGPRCount = Val;
6137 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6141 "directive is not supported with architected flat scratch",
6144 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6147 ImpliedUserSGPRCount += 4;
6148 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6151 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6154 return OutOfRangeError(ValRange);
6158 ImpliedUserSGPRCount += Val;
6159 PreloadLength = Val;
6161 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6164 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6167 return OutOfRangeError(ValRange);
6171 PreloadOffset = Val;
6172 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6175 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6178 ImpliedUserSGPRCount += 2;
6179 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6182 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6185 ImpliedUserSGPRCount += 2;
6186 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6189 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6192 ImpliedUserSGPRCount += 2;
6193 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6196 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6199 ImpliedUserSGPRCount += 2;
6200 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6203 "directive is not supported with architected flat scratch",
6207 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6210 ImpliedUserSGPRCount += 2;
6211 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6214 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6217 ImpliedUserSGPRCount += 1;
6218 }
else if (ID ==
".amdhsa_wavefront_size32") {
6221 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6222 EnableWavefrontSize32 = Val;
6224 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6226 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6228 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6230 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6233 "directive is not supported with architected flat scratch",
6236 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6238 }
else if (ID ==
".amdhsa_enable_private_segment") {
6242 "directive is not supported without architected flat scratch",
6245 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6247 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6249 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6251 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6253 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6255 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6257 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6259 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6261 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6263 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6265 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6267 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6268 VGPRRange = ValRange;
6269 NextFreeVGPR = ExprVal;
6270 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6271 SGPRRange = ValRange;
6272 NextFreeSGPR = ExprVal;
6273 }
else if (ID ==
".amdhsa_accum_offset") {
6275 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6276 AccumOffset = ExprVal;
6277 }
else if (ID ==
".amdhsa_named_barrier_count") {
6279 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6280 NamedBarCnt = ExprVal;
6281 }
else if (ID ==
".amdhsa_reserve_vcc") {
6283 return OutOfRangeError(ValRange);
6284 ReserveVCC = ExprVal;
6285 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6287 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6290 "directive is not supported with architected flat scratch",
6293 return OutOfRangeError(ValRange);
6294 ReserveFlatScr = ExprVal;
6295 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6297 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6299 return OutOfRangeError(ValRange);
6300 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6301 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6302 if (Val != XnackOn) {
6303 return getParser().Error(
6305 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6307 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6309 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6311 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6313 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6315 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6317 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6319 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6321 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6323 }
else if (ID ==
".amdhsa_dx10_clamp") {
6324 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6325 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6328 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6330 }
else if (ID ==
".amdhsa_ieee_mode") {
6331 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6332 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6335 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6337 }
else if (ID ==
".amdhsa_fp16_overflow") {
6339 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6341 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6343 }
else if (ID ==
".amdhsa_tg_split") {
6345 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6348 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6351 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6353 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6355 }
else if (ID ==
".amdhsa_memory_ordered") {
6357 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6359 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6361 }
else if (ID ==
".amdhsa_forward_progress") {
6363 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6365 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6367 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6369 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6370 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6372 SharedVGPRCount = Val;
6374 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6376 }
else if (ID ==
".amdhsa_inst_pref_size") {
6378 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6379 if (
ISA.Major == 11) {
6381 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6385 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6388 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6391 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6393 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6395 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6397 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6400 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6402 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6404 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6406 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6408 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6410 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6412 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6414 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6416 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6418 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6420 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6422 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6425 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6428#undef PARSE_BITS_ENTRY
6431 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6432 return TokError(
".amdhsa_next_free_vgpr directive is required");
6434 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6435 return TokError(
".amdhsa_next_free_sgpr directive is required");
6437 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6439 return TokError(
"too many user SGPRs enabled, found " +
6440 Twine(UserSGPRCount) +
", but only " +
6446 if (PreloadLength) {
6452 const MCExpr *VGPRBlocks;
6453 const MCExpr *SGPRBlocks;
6454 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6455 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6456 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6457 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6460 int64_t EvaluatedVGPRBlocks;
6461 bool VGPRBlocksEvaluatable =
6462 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6463 if (VGPRBlocksEvaluatable &&
6465 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6466 return OutOfRangeError(VGPRRange);
6470 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6471 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6473 int64_t EvaluatedSGPRBlocks;
6474 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6476 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6477 return OutOfRangeError(SGPRRange);
6480 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6481 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6483 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6484 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6485 "enabled user SGPRs");
6491 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6492 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6497 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6498 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6503 return TokError(
"Kernarg size should be resolvable");
6505 if (PreloadLength && kernarg_size &&
6506 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6507 return TokError(
"Kernarg preload length + offset is larger than the "
6508 "kernarg segment size");
6511 if (!Seen.
contains(
".amdhsa_accum_offset"))
6512 return TokError(
".amdhsa_accum_offset directive is required");
6513 int64_t EvaluatedAccum;
6514 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6515 uint64_t UEvaluatedAccum = EvaluatedAccum;
6516 if (AccumEvaluatable &&
6517 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6518 return TokError(
"accum_offset should be in range [4..256] in "
6521 int64_t EvaluatedNumVGPR;
6522 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6526 return TokError(
"accum_offset exceeds total VGPR allocation");
6532 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6533 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6539 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6540 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6543 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6545 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6546 return TokError(
"shared_vgpr_count directive not valid on "
6547 "wavefront size 32");
6550 if (VGPRBlocksEvaluatable &&
6551 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6553 return TokError(
"shared_vgpr_count*2 + "
6554 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6559 emitTargetDirective();
6560 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6561 NextFreeVGPR, NextFreeSGPR,
6562 ReserveVCC, ReserveFlatScr);
6566bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6568 if (ParseAsAbsoluteExpression(
Version))
6571 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6572 emitTargetDirective();
6576bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6577 AMDGPUMCKernelCodeT &
C) {
6580 if (ID ==
"max_scratch_backing_memory_byte_size") {
6581 Parser.eatToEndOfStatement();
6585 SmallString<40> ErrStr;
6586 raw_svector_ostream Err(ErrStr);
6587 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6588 return TokError(Err.
str());
6592 if (ID ==
"enable_wavefront_size32") {
6595 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6597 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6600 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6604 if (ID ==
"wavefront_size") {
6605 if (
C.wavefront_size == 5) {
6607 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6609 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6610 }
else if (
C.wavefront_size == 6) {
6612 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6619bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6620 AMDGPUMCKernelCodeT KernelCode;
6630 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6633 if (ID ==
".end_amd_kernel_code_t")
6636 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6641 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6646bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6647 StringRef KernelName;
6648 if (!parseId(KernelName,
"expected symbol name"))
6651 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6658bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6659 if (!getSTI().getTargetTriple().isAMDGCN()) {
6660 return Error(getLoc(),
6661 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6665 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6667 std::optional<AMDGPU::TargetID> MaybeParsed =
6670 return Error(getParser().getTok().getLoc(),
6671 "malformed target id '" + TargetIDDirective +
"'");
6674 const Triple &
TT = getSTI().getTargetTriple();
6680 return Error(getParser().getTok().getLoc(),
6681 "target id '" + TargetIDDirective +
6682 "' specifies a processor that is not valid for subarch '" +
6683 TT.getArchName() +
"'");
6686 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6687 getTargetStreamer().getTargetID();
6690 const Triple &STITriple = getSTI().getTargetTriple();
6691 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6692 return Error(getParser().getTok().getLoc(),
6693 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6694 " is incompatible with " +
6695 Twine(CurrentTargetID->toString()));
6699 StringRef DirectiveProcessor =
6702 if (DirectiveISA != ISA) {
6703 return Error(getParser().getTok().getLoc(),
6704 ".amd_amdgpu_isa directive processor " +
6705 Twine(DirectiveProcessor) +
6706 " does not match the specified processor " +
6707 Twine(getSTI().
getCPU()));
6710 getTargetStreamer().EmitISAVersion();
6716bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6719 std::string HSAMetadataString;
6724 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6725 return Error(getLoc(),
"invalid HSA metadata");
6732bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6733 const char *AssemblerDirectiveEnd,
6734 std::string &CollectString) {
6736 raw_string_ostream CollectStream(CollectString);
6738 getLexer().setSkipSpace(
false);
6740 bool FoundEnd =
false;
6743 CollectStream << getTokenStr();
6747 if (trySkipId(AssemblerDirectiveEnd)) {
6752 CollectStream << Parser.parseStringToEndOfStatement()
6753 <<
getContext().getAsmInfo().getSeparatorString();
6755 Parser.eatToEndOfStatement();
6758 getLexer().setSkipSpace(
true);
6761 return TokError(Twine(
"expected directive ") +
6762 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6769bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6775 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6776 if (!PALMetadata->setFromString(
String))
6777 return Error(getLoc(),
"invalid PAL metadata");
6782bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6785 Twine(
" directive is "
6786 "not available on non-amdpal OSes"))
6790 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6791 PALMetadata->setLegacy();
6794 if (ParseAsAbsoluteExpression(
Key)) {
6795 return TokError(Twine(
"invalid value in ") +
6799 return TokError(Twine(
"expected an even number of values in ") +
6802 if (ParseAsAbsoluteExpression(
Value)) {
6803 return TokError(Twine(
"invalid value in ") +
6806 PALMetadata->setRegister(
Key,
Value);
6815bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6816 if (getParser().checkForValidSection())
6820 SMLoc NameLoc = getLoc();
6821 if (getParser().parseIdentifier(Name))
6822 return TokError(
"expected identifier in directive");
6825 if (getParser().parseComma())
6831 SMLoc SizeLoc = getLoc();
6832 if (getParser().parseAbsoluteExpression(
Size))
6835 return Error(SizeLoc,
"size must be non-negative");
6836 if (
Size > LocalMemorySize)
6837 return Error(SizeLoc,
"size is too large");
6839 int64_t Alignment = 4;
6841 SMLoc AlignLoc = getLoc();
6842 if (getParser().parseAbsoluteExpression(Alignment))
6845 return Error(AlignLoc,
"alignment must be a power of two");
6850 if (Alignment >= 1u << 31)
6851 return Error(AlignLoc,
"alignment is too large");
6857 Symbol->redefineIfPossible();
6858 if (!
Symbol->isUndefined())
6859 return Error(NameLoc,
"invalid symbol redefinition");
6861 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6865bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6866 if (getParser().checkForValidSection())
6870 if (getParser().parseIdentifier(FuncName))
6871 return TokError(
"expected symbol name after .amdgpu_info");
6874 AMDGPU::InfoSectionData ParsedInfoData;
6875 AMDGPU::FuncInfo FI;
6877 bool HasScalarAttrs =
false;
6884 SMLoc IDLoc = getLoc();
6885 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6888 if (ID ==
".end_amdgpu_info")
6896 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6898 if (Dir ==
"flags") {
6900 if (getParser().parseAbsoluteExpression(Val))
6903 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6905 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6907 HasScalarAttrs =
true;
6908 }
else if (Dir ==
"num_sgpr") {
6910 if (getParser().parseAbsoluteExpression(Val))
6912 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6913 HasScalarAttrs =
true;
6914 }
else if (Dir ==
"num_vgpr") {
6916 if (getParser().parseAbsoluteExpression(Val))
6919 HasScalarAttrs =
true;
6920 }
else if (Dir ==
"num_agpr") {
6922 if (getParser().parseAbsoluteExpression(Val))
6925 HasScalarAttrs =
true;
6926 }
else if (Dir ==
"private_segment_size") {
6928 if (getParser().parseAbsoluteExpression(Val))
6931 HasScalarAttrs =
true;
6932 }
else if (Dir ==
"use") {
6934 if (getParser().parseIdentifier(ResName))
6935 return TokError(
"expected resource symbol for .amdgpu_use");
6936 ParsedInfoData.
Uses.push_back(
6937 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6938 }
else if (Dir ==
"call") {
6940 if (getParser().parseIdentifier(DstName))
6941 return TokError(
"expected callee symbol for .amdgpu_call");
6942 ParsedInfoData.
Calls.push_back(
6943 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6944 }
else if (Dir ==
"indirect_call") {
6946 if (getParser().parseEscapedString(TypeId))
6947 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6948 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6949 }
else if (Dir ==
"typeid") {
6951 if (getParser().parseEscapedString(TypeId))
6952 return TokError(
"expected type ID string for .amdgpu_typeid");
6953 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6955 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6960 ParsedInfoData.
Funcs.push_back(std::move(FI));
6962 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6963 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6964 Data.Funcs.push_back(std::move(Func));
6965 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6966 Data.Uses.push_back(Use);
6967 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6969 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6972 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6973 Data.TypeIds.push_back(std::move(TypeId));
6978void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
6985void AMDGPUAsmParser::checkKernelPrologues() {
6986 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
6987 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
6988 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
6989 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
6990 if (!AMDHSAKernelSymbols.
contains(Sym))
6992 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
6993 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
6997 "' does not begin with the required prologue "
6998 "sequence: s_mov_b64 followed by v_nop and "
6999 "global_prefetch_b8");
7003 OpcodeStream.
clear();
7004 OpcodeStreamSymbols.clear();
7005 AMDHSAKernelSymbols.
clear();
7008void AMDGPUAsmParser::onEndOfFile() {
7009 emitTargetDirective();
7010 checkKernelPrologues();
7012 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7015bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7016 StringRef IDVal = DirectiveID.
getString();
7019 if (IDVal ==
".amdhsa_kernel")
7020 return ParseDirectiveAMDHSAKernel();
7022 if (IDVal ==
".amdhsa_code_object_version")
7023 return ParseDirectiveAMDHSACodeObjectVersion();
7027 return ParseDirectiveHSAMetadata();
7029 if (IDVal ==
".amd_kernel_code_t")
7030 return ParseDirectiveAMDKernelCodeT();
7032 if (IDVal ==
".amdgpu_hsa_kernel")
7033 return ParseDirectiveAMDGPUHsaKernel();
7035 if (IDVal ==
".amd_amdgpu_isa")
7036 return ParseDirectiveISAVersion();
7040 Twine(
" directive is "
7041 "not available on non-amdhsa OSes"))
7046 if (IDVal ==
".amdgcn_target")
7047 return ParseDirectiveAMDGCNTarget();
7049 if (IDVal ==
".amdgpu_lds")
7050 return ParseDirectiveAMDGPULDS();
7052 if (IDVal ==
".amdgpu_info")
7053 return ParseDirectiveAMDGPUInfo();
7056 return ParseDirectivePALMetadataBegin();
7059 return ParseDirectivePALMetadata();
7064bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7071 return hasSGPR104_SGPR105();
7074 case SRC_SHARED_BASE_LO:
7075 case SRC_SHARED_BASE:
7076 case SRC_SHARED_LIMIT_LO:
7077 case SRC_SHARED_LIMIT:
7079 case SRC_PRIVATE_BASE_LO:
7080 case SRC_PRIVATE_BASE:
7081 case SRC_PRIVATE_LIMIT_LO:
7082 case SRC_PRIVATE_LIMIT:
7084 case SRC_FLAT_SCRATCH_BASE_LO:
7085 case SRC_FLAT_SCRATCH_BASE_HI:
7086 return hasGloballyAddressableScratch();
7087 case SRC_POPS_EXITING_WAVE_ID:
7100 getTargetStreamer().getTargetID()->isXnackSupported();
7130 return hasSGPR102_SGPR103();
7138 ParseStatus Res = parseVOPD(
Operands);
7143 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7155 SMLoc LBraceLoc = getLoc();
7160 auto Loc = getLoc();
7163 Error(Loc,
"expected a register");
7167 RBraceLoc = getLoc();
7172 "expected a comma or a closing square bracket"))
7176 if (
Operands.size() - Prefix > 1) {
7178 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7179 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7188StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7190 setForcedEncodingSize(0);
7191 setForcedDPP(
false);
7192 setForcedSDWA(
false);
7194 if (
Name.consume_back(
"_e64_dpp")) {
7196 setForcedEncodingSize(64);
7199 if (
Name.consume_back(
"_e64")) {
7200 setForcedEncodingSize(64);
7203 if (
Name.consume_back(
"_e32")) {
7204 setForcedEncodingSize(32);
7207 if (
Name.consume_back(
"_dpp")) {
7211 if (
Name.consume_back(
"_sdwa")) {
7212 setForcedSDWA(
true);
7220 unsigned VariantID);
7226 Name = parseMnemonicSuffix(Name);
7232 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7234 bool IsMIMG = Name.starts_with(
"image_");
7237 OperandMode
Mode = OperandMode_Default;
7239 Mode = OperandMode_NSA;
7243 checkUnsupportedInstruction(Name, NameLoc);
7244 if (!Parser.hasPendingError()) {
7247 :
"not a valid operand.";
7267ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7270 if (!trySkipId(Name))
7273 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7277ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7286ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7288 std::function<
bool(int64_t &)> ConvertResult) {
7292 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7296 if (ConvertResult && !ConvertResult(
Value)) {
7297 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7300 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7304ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7306 bool (*ConvertResult)(int64_t &)) {
7315 const unsigned MaxSize = 4;
7319 for (
int I = 0;; ++
I) {
7321 SMLoc Loc = getLoc();
7325 if (
Op != 0 &&
Op != 1)
7326 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7333 if (
I + 1 == MaxSize)
7334 return Error(getLoc(),
"expected a closing square bracket");
7340 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7344ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7346 AMDGPUOperand::ImmTy ImmTy,
7347 bool IgnoreNegative) {
7351 if (trySkipId(Name)) {
7353 }
else if (trySkipId(
"no", Name)) {
7362 return Error(S,
"r128 modifier is not supported on this GPU");
7363 if (Name ==
"a16" && !
hasA16())
7364 return Error(S,
"a16 modifier is not supported on this GPU");
7366 if (Bit == 0 && Name ==
"gds") {
7369 return Error(S,
"nogds is not allowed");
7372 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7373 ImmTy = AMDGPUOperand::ImmTyR128A16;
7375 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7379unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7380 bool &Disabling)
const {
7381 Disabling =
Id.consume_front(
"no");
7384 return StringSwitch<unsigned>(Id)
7391 return StringSwitch<unsigned>(Id)
7401 SMLoc StringLoc = getLoc();
7403 int64_t CPolVal = 0;
7423 ResScope = parseScope(
Operands, Scope);
7436 if (trySkipId(
"nv")) {
7440 }
else if (trySkipId(
"no",
"nv")) {
7447 if (trySkipId(
"scale_offset")) {
7451 }
else if (trySkipId(
"no",
"scale_offset")) {
7464 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7465 AMDGPUOperand::ImmTyCPol));
7470 SMLoc OpLoc = getLoc();
7471 unsigned Enabled = 0, Seen = 0;
7475 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7482 return Error(S,
"dlc modifier is not supported on this GPU");
7485 return Error(S,
"scc modifier is not supported on this GPU");
7488 return Error(S,
"duplicate cache policy modifier");
7500 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7509 ParseStatus Res = parseStringOrIntWithPrefix(
7510 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7524 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7528 if (
Value ==
"TH_DEFAULT")
7530 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7531 Value ==
"TH_LOAD_NT_WB") {
7532 return Error(StringLoc,
"invalid th value");
7533 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7535 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7537 }
else if (
Value.consume_front(
"TH_STORE_")) {
7540 return Error(StringLoc,
"invalid th value");
7543 if (
Value ==
"BYPASS")
7548 TH |= StringSwitch<int64_t>(
Value)
7558 .Default(0xffffffff);
7560 TH |= StringSwitch<int64_t>(
Value)
7571 .Default(0xffffffff);
7574 if (TH == 0xffffffff)
7575 return Error(StringLoc,
"invalid th value");
7582 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7583 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7584 std::optional<unsigned> InsertAt = std::nullopt) {
7585 auto i = OptionalIdx.find(ImmT);
7586 if (i != OptionalIdx.end()) {
7587 unsigned Idx = i->second;
7588 const AMDGPUOperand &
Op =
7589 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7593 Op.addImmOperands(Inst, 1);
7595 if (InsertAt.has_value())
7602ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7608 StringLoc = getLoc();
7613ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7619 SMLoc StringLoc = getLoc();
7623 Value = getTokenStr();
7627 if (
Value == Ids[IntVal])
7632 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7633 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7638ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7640 AMDGPUOperand::ImmTy
Type) {
7644 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7646 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7655bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7658 SMLoc Loc = getLoc();
7660 auto Res = parseIntWithPrefix(Pref, Val);
7666 if (Val < 0 || Val > MaxVal) {
7667 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7676 AMDGPUOperand::ImmTy ImmTy) {
7677 const char *Pref =
"index_key";
7679 SMLoc Loc = getLoc();
7680 auto Res = parseIntWithPrefix(Pref, ImmVal);
7684 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7685 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7686 (ImmVal < 0 || ImmVal > 1))
7687 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7689 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7690 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7692 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7697 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7701 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7705 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7710 AMDGPUOperand::ImmTy
Type) {
7716 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7717 AMDGPUOperand::ImmTyMatrixAFMT);
7721 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7722 AMDGPUOperand::ImmTyMatrixBFMT);
7727 AMDGPUOperand::ImmTy
Type) {
7733 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7734 AMDGPUOperand::ImmTyMatrixAScale);
7738 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7739 AMDGPUOperand::ImmTyMatrixBScale);
7744 AMDGPUOperand::ImmTy
Type) {
7750 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7751 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7755 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7756 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7761ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7762 using namespace llvm::AMDGPU::MTBUFFormat;
7768 for (
int I = 0;
I < 2; ++
I) {
7769 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7772 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7777 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7783 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7786 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7787 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7793ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7794 using namespace llvm::AMDGPU::MTBUFFormat;
7798 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7801 if (Fmt == UFMT_UNDEF)
7808bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7809 StringRef FormatStr, SMLoc Loc) {
7810 using namespace llvm::AMDGPU::MTBUFFormat;
7814 if (
Format != DFMT_UNDEF) {
7820 if (
Format != NFMT_UNDEF) {
7825 Error(Loc,
"unsupported format");
7829ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7832 using namespace llvm::AMDGPU::MTBUFFormat;
7836 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7841 SMLoc Loc = getLoc();
7842 if (!parseId(Str,
"expected a format string") ||
7843 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7845 if (Dfmt == DFMT_UNDEF)
7846 return Error(Loc,
"duplicate numeric format");
7847 if (Nfmt == NFMT_UNDEF)
7848 return Error(Loc,
"duplicate data format");
7851 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7852 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7856 if (Ufmt == UFMT_UNDEF)
7857 return Error(FormatLoc,
"unsupported format");
7866ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7869 using namespace llvm::AMDGPU::MTBUFFormat;
7872 if (Id == UFMT_UNDEF)
7876 return Error(Loc,
"unified format is not supported on this GPU");
7882ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7883 using namespace llvm::AMDGPU::MTBUFFormat;
7884 SMLoc Loc = getLoc();
7889 return Error(Loc,
"out of range format");
7894ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7895 using namespace llvm::AMDGPU::MTBUFFormat;
7901 StringRef FormatStr;
7902 SMLoc Loc = getLoc();
7903 if (!parseId(FormatStr,
"expected a format string"))
7906 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7908 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7918 return parseNumericFormat(
Format);
7922 using namespace llvm::AMDGPU::MTBUFFormat;
7926 SMLoc Loc = getLoc();
7936 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7955 Res = parseSymbolicOrNumericFormat(
Format);
7960 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7961 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7968 return Error(getLoc(),
"duplicate format");
7974 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7976 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7977 AMDGPUOperand::ImmTyInstOffset);
7984 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
7986 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
7992 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
7995 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8005 OptionalImmIndexMap OptionalIdx;
8007 unsigned OperandIdx[4];
8008 unsigned EnMask = 0;
8011 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8012 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8017 OperandIdx[SrcIdx] = Inst.
size();
8018 Op.addRegOperands(Inst, 1);
8025 OperandIdx[SrcIdx] = Inst.
size();
8031 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8032 Op.addImmOperands(Inst, 1);
8036 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8040 OptionalIdx[
Op.getImmTy()] = i;
8046 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8053 for (
auto i = 0; i < SrcIdx; ++i) {
8055 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8061 AMDGPUOperand::ImmTyExpCompr);
8071 int64_t CntVal,
bool Saturate,
8077 IntVal =
encode(ISA, IntVal, CntVal);
8078 if (CntVal !=
decode(ISA, IntVal)) {
8080 IntVal =
encode(ISA, IntVal, -1);
8088bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8090 SMLoc CntLoc = getLoc();
8091 StringRef CntName = getTokenStr();
8098 SMLoc ValLoc = getLoc();
8105 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8107 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8109 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8112 Error(CntLoc,
"invalid counter name " + CntName);
8117 Error(ValLoc,
"too large value for " + CntName);
8126 Error(getLoc(),
"expected a counter name");
8140 if (!parseCnt(Waitcnt))
8148 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8152bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8153 SMLoc FieldLoc = getLoc();
8154 StringRef FieldName = getTokenStr();
8159 SMLoc ValueLoc = getLoc();
8166 if (FieldName ==
"instid0") {
8168 }
else if (FieldName ==
"instskip") {
8170 }
else if (FieldName ==
"instid1") {
8173 Error(FieldLoc,
"invalid field name " + FieldName);
8192 .Case(
"VALU_DEP_1", 1)
8193 .Case(
"VALU_DEP_2", 2)
8194 .Case(
"VALU_DEP_3", 3)
8195 .Case(
"VALU_DEP_4", 4)
8196 .Case(
"TRANS32_DEP_1", 5)
8197 .Case(
"TRANS32_DEP_2", 6)
8198 .Case(
"TRANS32_DEP_3", 7)
8199 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8200 .Case(
"SALU_CYCLE_1", 9)
8201 .Case(
"SALU_CYCLE_2", 10)
8202 .Case(
"SALU_CYCLE_3", 11)
8210 Delay |=
Value << Shift;
8220 if (!parseDelay(Delay))
8228 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8232bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8234bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8240void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8241 StringRef DepCtrName) {
8244 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8247 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8250 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8253 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8260bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8262 using namespace llvm::AMDGPU::DepCtr;
8264 SMLoc DepCtrLoc = getLoc();
8265 StringRef DepCtrName = getTokenStr();
8275 unsigned PrevOprMask = UsedOprMask;
8276 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8279 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8288 Error(getLoc(),
"expected a counter name");
8293 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8294 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8299 using namespace llvm::AMDGPU::DepCtr;
8302 SMLoc Loc = getLoc();
8305 unsigned UsedOprMask = 0;
8307 if (!parseDepCtr(DepCtr, UsedOprMask))
8315 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8319bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8325ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8327 OperandInfoTy &Width) {
8328 using namespace llvm::AMDGPU::Hwreg;
8334 HwReg.Loc = getLoc();
8337 HwReg.IsSymbolic =
true;
8339 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8347 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8357 Width.Loc = getLoc();
8366 using namespace llvm::AMDGPU::Hwreg;
8369 SMLoc Loc = getLoc();
8371 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8373 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8374 HwregOffset::Default);
8375 struct : StructuredOpField {
8376 using StructuredOpField::StructuredOpField;
8377 bool validate(AMDGPUAsmParser &Parser)
const override {
8379 return Error(Parser,
"only values from 1 to 32 are legal");
8382 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8383 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8386 Res = parseHwregFunc(HwReg,
Offset, Width);
8389 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8391 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8395 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8402 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8404 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8408bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8414bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8415 OperandInfoTy &Stream) {
8416 using namespace llvm::AMDGPU::SendMsg;
8421 Msg.IsSymbolic =
true;
8428 Op.IsDefined =
true;
8434 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8439 Stream.IsDefined =
true;
8440 Stream.Loc = getLoc();
8449bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8450 const OperandInfoTy &
Op,
8451 const OperandInfoTy &Stream) {
8452 using namespace llvm::AMDGPU::SendMsg;
8461 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8466 Error(
Msg.Loc,
"invalid message id");
8472 Error(
Op.Loc,
"message does not support operations");
8474 Error(
Msg.Loc,
"missing message operation");
8480 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8482 Error(
Op.Loc,
"invalid operation id");
8487 Error(Stream.Loc,
"message operation does not support streams");
8491 Error(Stream.Loc,
"invalid message stream id");
8498 using namespace llvm::AMDGPU::SendMsg;
8501 SMLoc Loc = getLoc();
8505 OperandInfoTy
Op(OP_NONE_);
8506 OperandInfoTy Stream(STREAM_ID_NONE_);
8507 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8512 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8514 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8520 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8524bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8527 using namespace llvm::AMDGPU::WaitEvent;
8529 SMLoc Loc = getLoc();
8532 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8534 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8536 StructuredOpField *TargetBitfield =
8537 isGFX11() ? &DontWaitExportReady : &ExportReady;
8539 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8543 if (!validateStructuredOpFields({TargetBitfield}))
8545 ImmVal = TargetBitfield->Val;
8552 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8554 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8555 AMDGPUOperand::ImmTyWaitEvent));
8559bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8572 int Slot = StringSwitch<int>(Str)
8579 return Error(S,
"invalid interpolation slot");
8582 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8593 if (!Str.starts_with(
"attr"))
8594 return Error(S,
"invalid interpolation attribute");
8596 StringRef Chan = Str.take_back(2);
8597 int AttrChan = StringSwitch<int>(Chan)
8604 return Error(S,
"invalid or missing interpolation attribute channel");
8606 Str = Str.drop_back(2).drop_front(4);
8609 if (Str.getAsInteger(10, Attr))
8610 return Error(S,
"invalid or missing interpolation attribute number");
8613 return Error(S,
"out of bounds interpolation attribute number");
8618 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8619 Operands.push_back(AMDGPUOperand::CreateImm(
8620 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8629 using namespace llvm::AMDGPU::Exp;
8639 return Error(S, (Id == ET_INVALID)
8640 ?
"invalid exp target"
8641 :
"exp target is not supported on this GPU");
8644 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8652bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8656bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8661 return getTokenKind() ==
Kind;
8664StringRef AMDGPUAsmParser::getId()
const {
8668bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8676bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8678 StringRef Tok = getTokenStr();
8687bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8689 if (isId(Id) && peekToken().is(Kind)) {
8698 if (isToken(Kind)) {
8706 const StringRef ErrMsg) {
8707 if (!trySkipToken(Kind)) {
8708 Error(getLoc(), ErrMsg);
8714bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8718 if (Parser.parseExpression(Expr))
8721 if (Expr->evaluateAsAbsolute(
Imm))
8724 if (Expected.empty()) {
8725 Error(S,
"expected absolute expression");
8728 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8737 if (Parser.parseExpression(Expr))
8741 if (Expr->evaluateAsAbsolute(IntVal)) {
8742 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8744 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8749bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8751 Val =
getToken().getStringContents();
8755 Error(getLoc(), ErrMsg);
8759bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8761 Val = getTokenStr();
8765 if (!ErrMsg.
empty())
8766 Error(getLoc(), ErrMsg);
8770AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8772AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8775 : getLexer().peekTok(ShouldSkipSpace);
8779 auto TokCount = getLexer().peekTokens(Tokens);
8781 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8786 return getLexer().getKind();
8789SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8791StringRef AMDGPUAsmParser::getTokenStr()
const {
8795void AMDGPUAsmParser::lex() { Parser.Lex(); }
8797const AMDGPUOperand &
8799 int MCOpIdx)
const {
8801 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8802 if (TargetOp.getMCOpIdx() == MCOpIdx)
8809 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8813SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8818 int MCOpIdx)
const {
8819 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8822SMLoc AMDGPUAsmParser::getOperandLoc(
8823 std::function<
bool(
const AMDGPUOperand &)>
Test,
8825 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8826 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8828 return Op.getStartLoc();
8833SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8835 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8850 StringRef
Id = getTokenStr();
8851 SMLoc IdLoc = getLoc();
8857 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8858 if (
I == Fields.
end())
8859 return Error(IdLoc,
"unknown field");
8860 if ((*I)->IsDefined)
8861 return Error(IdLoc,
"duplicate field");
8864 (*I)->Loc = getLoc();
8867 (*I)->IsDefined =
true;
8874bool AMDGPUAsmParser::validateStructuredOpFields(
8876 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8877 return F->validate(*
this);
8887 const unsigned XorMask) {
8894bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8895 const unsigned MaxVal,
8896 const Twine &ErrMsg, SMLoc &Loc) {
8912bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8913 const unsigned MinVal,
8914 const unsigned MaxVal,
8915 const StringRef ErrMsg) {
8917 for (
unsigned i = 0; i < OpNum; ++i) {
8918 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8925bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8926 using namespace llvm::AMDGPU::Swizzle;
8929 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8930 "expected a 2-bit lane id")) {
8940bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8941 using namespace llvm::AMDGPU::Swizzle;
8947 if (!parseSwizzleOperand(GroupSize, 2, 32,
8948 "group size must be in the interval [2,32]", Loc)) {
8952 Error(Loc,
"group size must be a power of two");
8955 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8956 "lane id must be in the interval [0,group size - 1]",
8964bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8965 using namespace llvm::AMDGPU::Swizzle;
8970 if (!parseSwizzleOperand(GroupSize, 2, 32,
8971 "group size must be in the interval [2,32]", Loc)) {
8975 Error(Loc,
"group size must be a power of two");
8983bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
8984 using namespace llvm::AMDGPU::Swizzle;
8989 if (!parseSwizzleOperand(GroupSize, 1, 16,
8990 "group size must be in the interval [1,16]", Loc)) {
8994 Error(Loc,
"group size must be a power of two");
9002bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9003 using namespace llvm::AMDGPU::Swizzle;
9010 SMLoc StrLoc = getLoc();
9011 if (!parseString(Ctl)) {
9014 if (Ctl.
size() != BITMASK_WIDTH) {
9015 Error(StrLoc,
"expected a 5-character mask");
9019 unsigned AndMask = 0;
9020 unsigned OrMask = 0;
9021 unsigned XorMask = 0;
9023 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9027 Error(StrLoc,
"invalid mask");
9048bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9049 using namespace llvm::AMDGPU::Swizzle;
9052 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9058 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9059 "FFT swizzle must be in the interval [0," +
9060 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9068bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9069 using namespace llvm::AMDGPU::Swizzle;
9072 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9079 if (!parseSwizzleOperand(
Direction, 0, 1,
9080 "direction must be 0 (left) or 1 (right)", Loc))
9084 if (!parseSwizzleOperand(
9085 RotateSize, 0, ROTATE_MAX_SIZE,
9086 "number of threads to rotate must be in the interval [0," +
9087 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9092 (RotateSize << ROTATE_SIZE_SHIFT);
9096bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9098 SMLoc OffsetLoc = getLoc();
9104 Error(OffsetLoc,
"expected a 16-bit offset");
9110bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9111 using namespace llvm::AMDGPU::Swizzle;
9115 SMLoc ModeLoc = getLoc();
9118 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9119 Ok = parseSwizzleQuadPerm(
Imm);
9120 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9121 Ok = parseSwizzleBitmaskPerm(
Imm);
9122 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9123 Ok = parseSwizzleBroadcast(
Imm);
9124 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9125 Ok = parseSwizzleSwap(
Imm);
9126 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9127 Ok = parseSwizzleReverse(
Imm);
9128 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9129 Ok = parseSwizzleFFT(
Imm);
9130 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9131 Ok = parseSwizzleRotate(
Imm);
9133 Error(ModeLoc,
"expected a swizzle mode");
9136 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9146 if (trySkipId(
"offset")) {
9150 if (trySkipId(
"swizzle")) {
9151 Ok = parseSwizzleMacro(
Imm);
9153 Ok = parseSwizzleOffset(
Imm);
9158 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9165bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9171int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9173 using namespace llvm::AMDGPU::VGPRIndexMode;
9185 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9186 if (trySkipId(IdSymbolic[ModeId])) {
9194 ?
"expected a VGPR index mode or a closing parenthesis"
9195 :
"expected a VGPR index mode");
9200 Error(S,
"duplicate VGPR index mode");
9208 "expected a comma or a closing parenthesis"))
9217 using namespace llvm::AMDGPU::VGPRIndexMode;
9223 Imm = parseGPRIdxMacro();
9227 if (getParser().parseAbsoluteExpression(
Imm))
9230 return Error(S,
"invalid immediate: only 4-bit values are legal");
9234 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9238bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9249 if (isRegister() || isModifier())
9256 assert(Opr.isImm() || Opr.isExpr());
9257 SMLoc Loc = Opr.getStartLoc();
9261 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9262 Error(Loc,
"expected an absolute expression or a label");
9263 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9264 Error(Loc,
"expected a 16-bit signed jump offset");
9284 OptionalImmIndexMap OptionalIdx;
9285 unsigned FirstOperandIdx = 1;
9286 bool IsAtomicReturn =
false;
9292 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9293 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9297 Op.addRegOperands(Inst, 1);
9301 if (IsAtomicReturn && i == FirstOperandIdx)
9302 Op.addRegOperands(Inst, 1);
9307 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9308 Op.addImmOperands(Inst, 1);
9320 OptionalIdx[
Op.getImmTy()] = i;
9324 AMDGPUOperand::ImmTyOffset);
9336bool AMDGPUOperand::isSMRDOffset8()
const {
9340bool AMDGPUOperand::isSMEMOffset()
const {
9342 return isImmLiteral();
9345bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9380bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9381 if (BoundCtrl == 0 || BoundCtrl == 1) {
9389void AMDGPUAsmParser::onBeginOfFile() {
9390 if (!getParser().getStreamer().getTargetStreamer())
9393 if (!getTargetStreamer().getTargetID())
9394 getTargetStreamer().initializeTargetID(getSTI(),
9398void AMDGPUAsmParser::emitTargetDirective() {
9399 if (TargetDirectiveEmitted)
9401 TargetDirectiveEmitted =
true;
9403 if (!getParser().getStreamer().getTargetStreamer() ||
9408 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9417bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9421 StringRef TokenId = getTokenStr();
9422 AGVK VK = StringSwitch<AGVK>(TokenId)
9423 .Case(
"max", AGVK::AGVK_Max)
9424 .Case(
"min", AGVK::AGVK_Min)
9425 .Case(
"or", AGVK::AGVK_Or)
9426 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9427 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9428 .Case(
"alignto", AGVK::AGVK_AlignTo)
9429 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9430 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9431 .Default(AGVK::AGVK_None);
9440 if (Exprs.
empty()) {
9442 "empty " + Twine(TokenId) +
" expression");
9445 if (CommaCount + 1 != Exprs.
size()) {
9447 "mismatch of commas in " + Twine(TokenId) +
" expression");
9451 Expected && Exprs.
size() != Expected) {
9452 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9453 Twine(Expected) +
" operands");
9460 if (getParser().parseExpression(Expr, EndLoc))
9464 if (LastTokenWasComma)
9468 "unexpected token in " + Twine(TokenId) +
" expression");
9474 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9478 StringRef
Name = getTokenStr();
9479 if (Name ==
"mul") {
9480 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9484 if (Name ==
"div") {
9485 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9496 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9501 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9502 AMDGPU::OpName::src2};
9510 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9515 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9517 if (
DstOp.isReg() &&
9522 if ((OpSel & (1 << SrcNum)) != 0)
9528void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9535 OptionalImmIndexMap &OptionalIdx) {
9536 cvtVOP3P(Inst,
Operands, OptionalIdx);
9545 &&
Desc.NumOperands > (OpNum + 1)
9547 &&
Desc.operands()[OpNum + 1].RegClass != -1
9549 &&
Desc.getOperandConstraint(OpNum + 1,
9553void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9555 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9556 AMDGPU::OpName::src2};
9557 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9558 AMDGPU::OpName::src1_modifiers,
9559 AMDGPU::OpName::src2_modifiers};
9560 for (
int J = 0; J < 3; ++J) {
9561 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9567 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9570 if ((OpSel & (1 << J)) != 0)
9573 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9580void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9582 OptionalImmIndexMap OptionalIdx;
9587 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9588 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9592 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9594 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9595 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9596 Op.isInterpAttrChan()) {
9598 }
else if (
Op.isImmModifier()) {
9599 OptionalIdx[
Op.getImmTy()] =
I;
9607 AMDGPUOperand::ImmTyHigh);
9611 AMDGPUOperand::ImmTyClamp);
9615 AMDGPUOperand::ImmTyOModSI);
9620 AMDGPUOperand::ImmTyOpSel);
9621 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9624 cvtOpSelHelper(Inst, OpSel);
9629 OptionalImmIndexMap OptionalIdx;
9634 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9635 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9639 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9641 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9642 }
else if (
Op.isImmModifier()) {
9643 OptionalIdx[
Op.getImmTy()] =
I;
9651 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9654 AMDGPUOperand::ImmTyOpSel);
9657 AMDGPUOperand::ImmTyWaitEXP);
9663 cvtOpSelHelper(Inst, OpSel);
9666void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9668 OptionalImmIndexMap OptionalIdx;
9671 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9675 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9676 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9679 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9684 if (NumOperands == CbszOpIdx) {
9689 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9690 }
else if (
Op.isImmModifier()) {
9691 OptionalIdx[
Op.getImmTy()] =
I;
9693 Op.addRegOrImmOperands(Inst, 1);
9698 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9699 if (CbszIdx != OptionalIdx.end()) {
9700 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9704 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9705 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9706 if (BlgpIdx != OptionalIdx.end()) {
9707 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9718 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9719 if (OpselIdx != OptionalIdx.end()) {
9720 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9724 unsigned OpSelHi = 0;
9725 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9726 if (OpselHiIdx != OptionalIdx.end()) {
9727 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9730 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9731 AMDGPU::OpName::src1_modifiers};
9733 for (
unsigned J = 0; J < 2; ++J) {
9734 unsigned ModVal = 0;
9735 if (OpSel & (1 << J))
9737 if (OpSelHi & (1 << J))
9740 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9746 OptionalImmIndexMap &OptionalIdx) {
9751 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9752 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9756 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9758 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9759 }
else if (
Op.isImmModifier()) {
9760 OptionalIdx[
Op.getImmTy()] =
I;
9762 Op.addRegOrImmOperands(Inst, 1);
9768 AMDGPUOperand::ImmTyScaleSel);
9772 AMDGPUOperand::ImmTyClamp);
9778 AMDGPUOperand::ImmTyByteSel);
9783 AMDGPUOperand::ImmTyOModSI);
9790 auto *it = Inst.
begin();
9792 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9801 OptionalImmIndexMap OptionalIdx;
9802 cvtVOP3(Inst,
Operands, OptionalIdx);
9806 OptionalImmIndexMap &OptIdx) {
9811 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9812 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9813 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9814 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9815 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9816 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9817 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9818 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9819 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9820 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9829 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9830 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9833 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9834 if (BitOp3Idx != -1) {
9841 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9842 if (OpSelIdx != -1) {
9846 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9847 if (OpSelHiIdx != -1) {
9848 int DefaultVal =
IsPacked ? -1 : 0;
9854 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9855 if (MatrixAFMTIdx != -1) {
9857 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9861 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9862 if (MatrixBFMTIdx != -1) {
9864 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9867 int MatrixAScaleIdx =
9868 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9869 if (MatrixAScaleIdx != -1) {
9871 AMDGPUOperand::ImmTyMatrixAScale, 0);
9874 int MatrixBScaleIdx =
9875 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9876 if (MatrixBScaleIdx != -1) {
9878 AMDGPUOperand::ImmTyMatrixBScale, 0);
9881 int MatrixAScaleFmtIdx =
9882 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9883 if (MatrixAScaleFmtIdx != -1) {
9885 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9888 int MatrixBScaleFmtIdx =
9889 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9890 if (MatrixBScaleFmtIdx != -1) {
9892 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9897 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9901 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9903 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9907 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9911 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9912 AMDGPU::OpName::src2};
9913 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9914 AMDGPU::OpName::src1_modifiers,
9915 AMDGPU::OpName::src2_modifiers};
9918 unsigned OpSelHi = 0;
9925 if (OpSelHiIdx != -1)
9934 for (
int J = 0; J < 3; ++J) {
9935 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9939 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9949 uint32_t ModVal = 0;
9951 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9952 if (SrcOp.
isReg() && getMRI()
9959 if ((OpSel & (1 << J)) != 0)
9963 if ((OpSelHi & (1 << J)) != 0)
9966 if ((NegLo & (1 << J)) != 0)
9969 if ((NegHi & (1 << J)) != 0)
9977 OptionalImmIndexMap OptIdx;
9983 unsigned i,
unsigned Opc,
9985 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
9986 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
9988 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
9994 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9997 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9998 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10000 OptionalImmIndexMap OptIdx;
10001 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10002 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10003 OptIdx[
Op.getImmTy()] = i;
10008 AMDGPUOperand::ImmTyIndexKey8bit);
10012 AMDGPUOperand::ImmTyIndexKey16bit);
10016 AMDGPUOperand::ImmTyIndexKey32bit);
10033 SMLoc S = getLoc();
10036 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10037 SMLoc OpYLoc = getLoc();
10040 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10043 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10052 auto addOp = [&](uint16_t ParsedOprIdx) {
10053 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10055 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10059 Op.addRegOperands(Inst, 1);
10063 Op.addImmOperands(Inst, 1);
10075 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10079 const auto &CInfo = InstInfo[CompIdx];
10080 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10081 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10082 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10083 if (CInfo.hasSrc2Acc())
10084 addOp(CInfo.getIndexOfDstInParsedOperands());
10088 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10089 if (BitOp3Idx != -1) {
10090 OptionalImmIndexMap OptIdx;
10091 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10093 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10103bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10105bool AMDGPUOperand::isDPPCtrl()
const {
10106 using namespace AMDGPU::DPP;
10108 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10111 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10112 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10113 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10114 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10115 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10116 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10117 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10118 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10119 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10120 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10121 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10122 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10131bool AMDGPUOperand::isBLGP()
const {
10135bool AMDGPUOperand::isS16Imm()
const {
10139bool AMDGPUOperand::isU16Imm()
const {
10147bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10152 SMLoc Loc =
getToken().getEndLoc();
10153 Token = std::string(getTokenStr());
10155 if (getLoc() != Loc)
10160 if (!parseId(Suffix))
10164 StringRef DimId = Token;
10179 SMLoc S = getLoc();
10185 SMLoc Loc = getLoc();
10186 if (!parseDimId(Encoding))
10187 return Error(Loc,
"invalid dim value");
10190 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10199 SMLoc S = getLoc();
10208 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10211 for (
size_t i = 0; i < 8; ++i) {
10215 SMLoc Loc = getLoc();
10216 if (getParser().parseAbsoluteExpression(Sels[i]))
10218 if (0 > Sels[i] || 7 < Sels[i])
10219 return Error(Loc,
"expected a 3-bit value");
10222 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10226 for (
size_t i = 0; i < 8; ++i)
10227 DPP8 |= (Sels[i] << (i * 3));
10230 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10234bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10236 if (Ctrl ==
"row_newbcast")
10239 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10242 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10243 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10246 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10247 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10251int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10254 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10258 for (
int i = 0; i < 4; ++i) {
10263 SMLoc Loc = getLoc();
10264 if (getParser().parseAbsoluteExpression(Temp))
10266 if (Temp < 0 || Temp > 3) {
10267 Error(Loc,
"expected a 2-bit value");
10271 Val += (Temp << i * 2);
10274 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10280int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10281 using namespace AMDGPU::DPP;
10286 SMLoc Loc = getLoc();
10288 if (getParser().parseAbsoluteExpression(Val))
10291 struct DppCtrlCheck {
10297 DppCtrlCheck
Check =
10298 StringSwitch<DppCtrlCheck>(Ctrl)
10299 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10300 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10301 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10302 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10303 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10304 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10305 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10306 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10307 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10308 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10312 if (
Check.Ctrl == -1) {
10313 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10321 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10329 using namespace AMDGPU::DPP;
10332 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10335 SMLoc S = getLoc();
10341 if (Ctrl ==
"row_mirror") {
10342 Val = DppCtrl::ROW_MIRROR;
10343 }
else if (Ctrl ==
"row_half_mirror") {
10344 Val = DppCtrl::ROW_HALF_MIRROR;
10347 if (Ctrl ==
"quad_perm") {
10348 Val = parseDPPCtrlPerm();
10350 Val = parseDPPCtrlSel(Ctrl);
10359 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10365 OptionalImmIndexMap OptionalIdx;
10372 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10374 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10375 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10379 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10380 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10384 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10385 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10386 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10387 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10388 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10389 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10390 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10391 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10392 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10398 if (OldIdx == NumOperands) {
10400 constexpr int DST_IDX = 0;
10402 }
else if (Src2ModIdx == NumOperands) {
10412 if (IsVOP3CvtSrDpp) {
10421 if (TiedTo != -1) {
10426 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10428 if (IsDPP8 &&
Op.isDppFI()) {
10431 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10432 }
else if (
Op.isReg()) {
10433 Op.addRegOperands(Inst, 1);
10434 }
else if (
Op.isImm() &&
10436 Op.addImmOperands(Inst, 1);
10437 }
else if (
Op.isImm()) {
10438 OptionalIdx[
Op.getImmTy()] =
I;
10446 AMDGPUOperand::ImmTyClamp);
10452 AMDGPUOperand::ImmTyByteSel);
10457 AMDGPUOperand::ImmTyOModSI);
10460 cvtVOP3P(Inst,
Operands, OptionalIdx);
10462 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10465 AMDGPUOperand::ImmTyOpSel);
10470 AMDGPUOperand::ImmTyDPP8);
10471 using namespace llvm::AMDGPU::DPP;
10475 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10477 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10479 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10481 AMDGPUOperand::ImmTyDppBoundCtrl);
10485 AMDGPUOperand::ImmTyDppFI);
10491 OptionalImmIndexMap OptionalIdx;
10495 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10496 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10503 if (TiedTo != -1) {
10508 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10510 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10518 Op.addImmOperands(Inst, 1);
10520 Op.addRegWithFPInputModsOperands(Inst, 2);
10521 }
else if (
Op.isDppFI()) {
10523 }
else if (
Op.isReg()) {
10524 Op.addRegOperands(Inst, 1);
10530 Op.addRegWithFPInputModsOperands(Inst, 2);
10531 }
else if (
Op.isReg()) {
10532 Op.addRegOperands(Inst, 1);
10533 }
else if (
Op.isDPPCtrl()) {
10534 Op.addImmOperands(Inst, 1);
10535 }
else if (
Op.isImm()) {
10537 OptionalIdx[
Op.getImmTy()] =
I;
10545 using namespace llvm::AMDGPU::DPP;
10549 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10551 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10553 AMDGPUOperand::ImmTyDppBoundCtrl);
10556 AMDGPUOperand::ImmTyDppFI);
10567 AMDGPUOperand::ImmTy
Type) {
10568 return parseStringOrIntWithPrefix(
10570 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10575 return parseStringOrIntWithPrefix(
10576 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10577 AMDGPUOperand::ImmTySDWADstUnused);
10581 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10585 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10588void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10590 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10593void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10595 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10603 SDWAInstType BasicInstType,
bool SkipDstVcc,
10605 using namespace llvm::AMDGPU::SDWA;
10607 OptionalImmIndexMap OptionalIdx;
10608 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10609 bool SkippedVcc =
false;
10613 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10614 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10618 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10619 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10620 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10626 if (BasicInstType == SDWAInstType::VOP2 &&
10632 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10638 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10639 }
else if (
Op.isImm()) {
10641 OptionalIdx[
Op.getImmTy()] =
I;
10645 SkippedVcc =
false;
10649 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10650 Opc != AMDGPU::V_NOP_sdwa_vi) {
10652 switch (BasicInstType) {
10653 case SDWAInstType::VOP1:
10656 AMDGPUOperand::ImmTyClamp, 0);
10660 AMDGPUOperand::ImmTyOModSI, 0);
10664 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10668 AMDGPUOperand::ImmTySDWADstUnused,
10669 DstUnused::UNUSED_PRESERVE);
10672 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10675 case SDWAInstType::VOP2:
10677 AMDGPUOperand::ImmTyClamp, 0);
10681 AMDGPUOperand::ImmTyOModSI, 0);
10684 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10686 AMDGPUOperand::ImmTySDWADstUnused,
10687 DstUnused::UNUSED_PRESERVE);
10689 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10691 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10694 case SDWAInstType::VOPC:
10697 AMDGPUOperand::ImmTyClamp, 0);
10699 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10701 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10708 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10709 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10710 auto *it = Inst.
begin();
10712 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10725#define GET_MATCHER_IMPLEMENTATION
10726#define GET_MNEMONIC_SPELL_CHECKER
10727#define GET_MNEMONIC_CHECKER
10728#include "AMDGPUGenAsmMatcher.inc"
10734 return parseTokenOp(
"addr64",
Operands);
10736 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10738 return parseTokenOp(
"idxen",
Operands);
10740 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10743 return parseTokenOp(
"offen",
Operands);
10745 return parseTokenOp(
"off",
Operands);
10746 case MCK_row_95_en:
10747 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10749 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10751 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10753 return tryCustomParseOperand(
Operands, MCK);
10758unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10764 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10767 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10769 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10771 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10773 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10775 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10777 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10779 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10780 case MCK_row_95_en:
10781 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10789 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10791 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10792 case MCK_SOPPBrTarget:
10793 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10794 case MCK_VReg32OrOff:
10795 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10796 case MCK_InterpSlot:
10797 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10798 case MCK_InterpAttr:
10799 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10800 case MCK_InterpAttrChan:
10801 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10803 case MCK_SReg_64_XEXEC:
10813 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10815 return Match_InvalidOperand;
10824 SMLoc S = getLoc();
10833 return Error(S,
"expected a 16-bit value");
10836 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10840bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10846bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
StringSet - A set-like wrapper for the StringMap.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
const MCRegisterClass & getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
bool isGFX10_BEncoding(const MCSubtargetInfo &STI)
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, int32_t ArgNumVGPR)
bool isGFX10(const MCSubtargetInfo &STI)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
bool isGFX13(const MCSubtargetInfo &STI)
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, OpName NamedIdx)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool isGFX13Plus(const MCSubtargetInfo &STI)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isGFX1250(const MCSubtargetInfo &STI)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
@ Default
The result value is uniform if and only if all operands are uniform.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0