70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
247 bool isRegOrInlineTarget(
unsigned TargetRCIdx, MVT type)
const {
248 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
252 return isRegOrInline(RCID, type) || isLiteralImm(type);
255 bool isRegOrImmWithInputModsTarget(
unsigned TargetRCIdx, MVT type)
const {
256 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
259 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
261 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
264 bool isRegOrImmWithInt32InputMods()
const {
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
278 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFPT16_LO16InputMods()
const {
294 bool isRegOrImmWithFP32InputMods()
const {
298 bool isRegOrImmWithFP64InputMods()
const {
299 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
302 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
303 return isRegOrInline(
304 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
307 bool isRegOrInlineImmWithFP32InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
311 bool isRegOrInlineImmWithFP64InputMods()
const {
312 return isRegOrInlineTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
315 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
317 bool isVRegWithFP32InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
321 bool isVRegWithFP64InputMods()
const {
322 return isRegClassTarget(AMDGPU::VReg_64_AlignTarget);
325 bool isPackedFP16InputMods()
const {
329 bool isPackedVGPRFP32InputMods()
const {
333 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
335 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
337 bool isRsrcReg32()
const {
return isRegClass(AMDGPU::RsrcReg32RegClassID); }
339 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
341 bool isAV_LdSt_32_Align2_RegOp()
const {
342 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
343 isRegClass(AMDGPU::AGPR_32RegClassID);
346 bool isVRegWithInputMods()
const;
347 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
348 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
349 bool isT16_LO16VRegWithInputMods()
const;
351 bool isSDWAOperand(MVT type)
const;
352 bool isSDWAFP16Operand()
const;
353 bool isSDWAFP32Operand()
const;
354 bool isSDWAInt16Operand()
const;
355 bool isSDWAInt32Operand()
const;
357 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
359 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
361 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
363 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
365 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
366 bool isDim()
const {
return isImmTy(ImmTyDim); }
367 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
368 bool isOff()
const {
return isImmTy(ImmTyOff); }
369 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
370 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
371 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
372 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
373 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
374 bool isFlatOffset()
const {
375 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
377 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
378 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
379 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
380 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
381 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
382 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
383 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
384 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
385 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
386 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
387 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
388 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
389 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
390 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
391 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
392 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
393 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
394 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
395 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
396 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
397 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
398 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
399 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
400 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
401 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
402 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
403 bool isDone()
const {
return isImmTy(ImmTyDone); }
404 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
406 bool isRegOrImm()
const {
return isReg() || isImm(); }
408 bool isRegClass(
unsigned RCID)
const;
411 bool isRegClassTarget(
unsigned TargetRCIdx)
const;
415 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
416 return isRegOrInline(RCID, type) && !hasModifiers();
419 bool isRegOrInlineNoModsTarget(
unsigned TargetRCIdx, MVT type)
const {
420 return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers();
423 bool isSCSrcB16()
const {
424 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
427 bool isSCSrc_b32()
const {
428 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
431 bool isSCSrc_b64()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
435 bool isBoolReg()
const;
437 bool isSSrc_b32()
const {
438 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
441 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
443 bool isSSrc_b64()
const {
446 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
447 (((
const MCTargetAsmParser *)AsmParser)
448 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
452 bool isSSrc_f32()
const {
453 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
456 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
458 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
460 bool isSSrc_NoInline_f16()
const {
return isSSrc_f16(); }
462 bool isSSrcOrLds_b32()
const {
463 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
464 isLiteralImm(MVT::i32) || isExpr();
467 bool isVCSrc_b32()
const {
468 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
471 bool isVCSrc_b32_Lo256()
const {
472 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
475 bool isVCSrc_b64_Lo256()
const {
476 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
479 bool isVCSrc_b64()
const {
480 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
483 bool isVCSrcT_b16()
const {
484 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
487 bool isVCSrcTB16_Lo128()
const {
488 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
491 bool isVCSrcFake16B16_Lo128()
const {
492 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
495 bool isVCSrc_b16()
const {
496 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
499 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
501 bool isVCSrc_f32()
const {
502 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
505 bool isVCSrc_f64()
const {
506 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
509 bool isVCSrcTBF16()
const {
510 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
513 bool isVCSrcT_f16()
const {
514 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
517 bool isVCSrcT_bf16()
const {
518 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
521 bool isVCSrcTF16_LO16()
const {
522 return isRegOrInlineNoMods(AMDGPU::VS_16_LO16RegClassID, MVT::f16);
525 bool isVCSrcTBF16_Lo128()
const {
526 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
529 bool isVCSrcTF16_Lo128()
const {
530 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
533 bool isVCSrcFake16BF16_Lo128()
const {
534 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
537 bool isVCSrcFake16F16_Lo128()
const {
538 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
541 bool isVCSrc_bf16()
const {
542 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
545 bool isVCSrc_f16()
const {
546 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
549 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
551 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
553 bool isVSrc_b32()
const {
554 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
557 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
559 bool isVSrc_v2b64()
const {
560 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
561 isLiteralImm(MVT::i64);
564 bool isVSrc_v2f64()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
566 isLiteralImm(MVT::f64);
569 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
571 bool isVSrcT_b16_Lo128()
const {
572 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
575 bool isVSrcFake16_b16_Lo128()
const {
576 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
579 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
581 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
583 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
585 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
587 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
589 bool isVSrc_f32()
const {
590 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
593 bool isVSrc_f64()
const {
594 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64) ||
595 isLiteralImm(MVT::f64);
598 bool isVSrcT_bf16()
const {
599 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
602 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
604 bool isVSrcT_f16_LO16()
const {
605 return isVCSrcTF16_LO16() || isLiteralImm(MVT::f16);
608 bool isVSrcT_bf16_Lo128()
const {
609 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
612 bool isVSrcT_f16_Lo128()
const {
613 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
616 bool isVSrcFake16_bf16_Lo128()
const {
617 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
620 bool isVSrcFake16_f16_Lo128()
const {
621 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
624 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
626 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
628 bool isVSrc_v2bf16()
const {
629 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
632 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
634 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
636 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
638 bool isVISrc_64_bf16()
const {
639 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::bf16);
642 bool isVISrc_64_f16()
const {
643 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f16);
646 bool isVISrc_64_b32()
const {
647 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::i32);
650 bool isVISrc_64_f64()
const {
651 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f64);
654 bool isVISrc_256_b32()
const {
655 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::i32);
658 bool isVISrc_256_f32()
const {
659 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f32);
662 bool isVISrc_256_f64()
const {
663 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f64);
666 bool isVISrc_512_f64()
const {
667 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f64);
670 bool isVISrc_128_b32()
const {
671 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::i32);
674 bool isVISrc_128_f32()
const {
675 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f32);
678 bool isVISrc_512_b32()
const {
679 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::i32);
682 bool isVISrc_512_f32()
const {
683 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f32);
686 bool isVISrc_1024_b32()
const {
687 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::i32);
690 bool isVISrc_1024_f32()
const {
691 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::f32);
694 bool isAISrc_64_f64()
const {
695 return isRegOrInlineNoModsTarget(AMDGPU::AReg_64_AlignTarget, MVT::f64);
698 bool isAISrc_128_b32()
const {
699 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::i32);
702 bool isAISrc_128_f32()
const {
703 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::f32);
706 bool isVISrc_128_bf16()
const {
707 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::bf16);
710 bool isVISrc_128_f16()
const {
711 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f16);
714 bool isAISrc_256_f64()
const {
715 return isRegOrInlineNoModsTarget(AMDGPU::AReg_256_AlignTarget, MVT::f64);
718 bool isAISrc_512_b32()
const {
719 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::i32);
722 bool isAISrc_512_f32()
const {
723 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::f32);
726 bool isAISrc_1024_b32()
const {
727 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::i32);
730 bool isAISrc_1024_f32()
const {
731 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::f32);
734 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
736 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
738 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
740 bool isMem()
const override {
return false; }
742 bool isExpr()
const {
return Kind == Expression; }
744 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
746 bool isSWaitCnt()
const;
747 bool isDepCtr()
const;
748 bool isSDelayALU()
const;
749 bool isHwreg()
const;
750 bool isSendMsg()
const;
751 bool isWaitEvent()
const;
752 bool isSplitBarrier()
const;
753 bool isSwizzle()
const;
754 bool isSMRDOffset8()
const;
755 bool isSMEMOffset()
const;
756 bool isSMRDLiteralOffset()
const;
758 bool isDPPCtrl()
const;
760 bool isGPRIdxMode()
const;
761 bool isS16Imm()
const;
762 bool isU16Imm()
const;
763 bool isEndpgm()
const;
765 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
766 return [
this,
P]() {
return P(*
this); };
771 return StringRef(Tok.Data, Tok.Length);
779 void setImm(int64_t Val) {
784 ImmTy getImmTy()
const {
789 MCRegister
getReg()
const override {
794 SMLoc getStartLoc()
const override {
return StartLoc; }
796 SMLoc getEndLoc()
const override {
return EndLoc; }
798 int getMCOpIdx()
const {
return MCOpIdx; }
800 Modifiers getModifiers()
const {
801 assert(isRegKind() || isImmTy(ImmTyNone));
802 return isRegKind() ?
Reg.Mods :
Imm.Mods;
805 void setModifiers(Modifiers Mods) {
806 assert(isRegKind() || isImmTy(ImmTyNone));
813 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
815 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
817 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
819 bool isForcedLit()
const {
820 return isImmLiteral() && getModifiers().isForcedLit();
823 bool isForcedLit64()
const {
824 return isImmLiteral() && getModifiers().isForcedLit64();
829 void addImmOperands(MCInst &Inst,
unsigned N,
830 bool ApplyModifiers =
true)
const;
832 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
833 bool ApplyModifiers)
const;
835 void addRegOperands(MCInst &Inst,
unsigned N)
const;
837 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
839 addRegOperands(Inst,
N);
841 addImmOperands(Inst,
N);
844 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
845 Modifiers Mods = getModifiers();
848 addRegOperands(Inst,
N);
850 addImmOperands(Inst,
N,
false);
854 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
855 assert(!hasIntModifiers());
856 addRegOrImmWithInputModsOperands(Inst,
N);
859 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
860 Modifiers Mods = getModifiers();
863 addRegOperands(Inst,
N);
866 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
867 assert(!hasIntModifiers());
868 addRegWithInputModsOperands(Inst,
N);
871 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
874 case ImmTyNone: OS <<
"None";
break;
875 case ImmTyGDS: OS <<
"GDS";
break;
876 case ImmTyLDS: OS <<
"LDS";
break;
877 case ImmTyOffen: OS <<
"Offen";
break;
878 case ImmTyIdxen: OS <<
"Idxen";
break;
879 case ImmTyAddr64: OS <<
"Addr64";
break;
880 case ImmTyOffset: OS <<
"Offset";
break;
881 case ImmTyInstOffset: OS <<
"InstOffset";
break;
882 case ImmTyOffset0: OS <<
"Offset0";
break;
883 case ImmTyOffset1: OS <<
"Offset1";
break;
884 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
885 case ImmTyCPol: OS <<
"CPol";
break;
886 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
887 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
888 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
889 case ImmTyTFE: OS <<
"TFE";
break;
890 case ImmTyIsAsync: OS <<
"IsAsync";
break;
891 case ImmTyD16: OS <<
"D16";
break;
892 case ImmTyFORMAT: OS <<
"FORMAT";
break;
893 case ImmTyClamp: OS <<
"Clamp";
break;
894 case ImmTyOModSI: OS <<
"OModSI";
break;
895 case ImmTyDPP8: OS <<
"DPP8";
break;
896 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
897 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
898 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
899 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
900 case ImmTyDppFI: OS <<
"DppFI";
break;
901 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
902 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
903 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
904 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
905 case ImmTyDMask: OS <<
"DMask";
break;
906 case ImmTyDim: OS <<
"Dim";
break;
907 case ImmTyUNorm: OS <<
"UNorm";
break;
908 case ImmTyDA: OS <<
"DA";
break;
909 case ImmTyR128A16: OS <<
"R128A16";
break;
910 case ImmTyA16: OS <<
"A16";
break;
911 case ImmTyLWE: OS <<
"LWE";
break;
912 case ImmTyOff: OS <<
"Off";
break;
913 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
914 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
915 case ImmTyExpVM: OS <<
"ExpVM";
break;
916 case ImmTyDone: OS <<
"Done";
break;
917 case ImmTyRowEn: OS <<
"RowEn";
break;
918 case ImmTyHwreg: OS <<
"Hwreg";
break;
919 case ImmTySendMsg: OS <<
"SendMsg";
break;
920 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
921 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
922 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
923 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
924 case ImmTyOpSel: OS <<
"OpSel";
break;
925 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
926 case ImmTyNegLo: OS <<
"NegLo";
break;
927 case ImmTyNegHi: OS <<
"NegHi";
break;
928 case ImmTySwizzle: OS <<
"Swizzle";
break;
929 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
930 case ImmTyHigh: OS <<
"High";
break;
931 case ImmTyBLGP: OS <<
"BLGP";
break;
932 case ImmTyCBSZ: OS <<
"CBSZ";
break;
933 case ImmTyABID: OS <<
"ABID";
break;
934 case ImmTyEndpgm: OS <<
"Endpgm";
break;
935 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
936 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
937 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
938 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
939 case ImmTyBitOp3: OS <<
"BitOp3";
break;
940 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
941 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
942 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
943 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
944 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
945 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
946 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
947 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
948 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
949 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
954 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
958 <<
" mods: " <<
Reg.Mods <<
'>';
962 if (getImmTy() != ImmTyNone) {
964 printImmTy(OS, getImmTy());
966 OS <<
" mods: " <<
Imm.Mods <<
'>';
979 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
980 int64_t Val, SMLoc Loc,
981 ImmTy
Type = ImmTyNone,
982 bool IsFPImm =
false) {
983 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
985 Op->Imm.IsFPImm = IsFPImm;
987 Op->Imm.Mods = Modifiers();
993 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
994 StringRef Str, SMLoc Loc,
995 bool HasExplicitEncodingSize =
true) {
996 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
997 Res->Tok.Data = Str.data();
998 Res->Tok.Length = Str.size();
1004 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1005 MCRegister
Reg, SMLoc S, SMLoc
E) {
1006 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1007 Op->Reg.RegNo =
Reg;
1008 Op->Reg.Mods = Modifiers();
1014 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1015 const class MCExpr *Expr, SMLoc S) {
1016 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1025 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1034#define GET_REGISTER_MATCHER
1035#include "AMDGPUGenAsmMatcher.inc"
1036#undef GET_REGISTER_MATCHER
1037#undef GET_SUBTARGET_FEATURE_NAME
1042class KernelScopeInfo {
1043 int SgprIndexUnusedMin = -1;
1044 int VgprIndexUnusedMin = -1;
1045 int AgprIndexUnusedMin = -1;
1049 void usesSgprAt(
int i) {
1050 if (i >= SgprIndexUnusedMin) {
1051 SgprIndexUnusedMin = ++i;
1054 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1060 void usesVgprAt(
int i) {
1061 if (i >= VgprIndexUnusedMin) {
1062 VgprIndexUnusedMin = ++i;
1065 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1067 VgprIndexUnusedMin);
1073 void usesAgprAt(
int i) {
1078 if (i >= AgprIndexUnusedMin) {
1079 AgprIndexUnusedMin = ++i;
1082 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1087 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1089 VgprIndexUnusedMin);
1096 KernelScopeInfo() =
default;
1100 MSTI = Ctx->getSubtargetInfo();
1102 usesSgprAt(SgprIndexUnusedMin = -1);
1103 usesVgprAt(VgprIndexUnusedMin = -1);
1105 usesAgprAt(AgprIndexUnusedMin = -1);
1109 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1110 unsigned RegWidth) {
1113 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1116 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1119 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1128 MCAsmParser &Parser;
1130 unsigned ForcedEncodingSize = 0;
1131 bool ForcedDPP =
false;
1132 bool ForcedSDWA =
false;
1133 KernelScopeInfo KernelScope;
1134 const unsigned HwMode;
1136 const AMDGPU::IsaVersion ISA;
1141#define GET_ASSEMBLER_HEADER
1142#include "AMDGPUGenAsmMatcher.inc"
1147 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1149 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1153 std::optional<AMDGPU::InfoSectionData> InfoData;
1162 SmallVector<unsigned> OpcodeStream;
1164 OpcodeStreamSymbols;
1165 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1168 void checkKernelPrologues();
1171 void createConstantSymbol(StringRef Id, int64_t Val);
1173 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1174 bool OutOfRangeError(SMRange
Range);
1190 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1191 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1192 std::optional<bool> EnableWavefrontSize32,
1193 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1194 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1195 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1196 bool ParseDirectiveAMDGCNTarget();
1197 bool ParseDirectiveAMDHSACodeObjectVersion();
1198 bool ParseDirectiveAMDHSAKernel();
1199 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1200 bool ParseDirectiveAMDKernelCodeT();
1202 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1203 bool ParseDirectiveAMDGPUHsaKernel();
1205 bool ParseDirectiveISAVersion();
1206 bool ParseDirectiveHSAMetadata();
1207 bool ParseDirectivePALMetadataBegin();
1208 bool ParseDirectivePALMetadata();
1209 bool ParseDirectiveAMDGPULDS();
1210 bool ParseDirectiveAMDGPUInfo();
1214 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1215 const char *AssemblerDirectiveEnd,
1216 std::string &CollectString);
1218 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1219 RegisterKind RegKind, MCRegister Reg1,
1220 RegisterKind RegKind1, SMLoc Loc);
1221 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1222 unsigned &RegNum,
unsigned &RegWidth,
1223 bool RestoreOnFailure =
false);
1224 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1225 unsigned &RegNum,
unsigned &RegWidth,
1226 SmallVectorImpl<AsmToken> &Tokens);
1227 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1229 SmallVectorImpl<AsmToken> &Tokens);
1230 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1232 SmallVectorImpl<AsmToken> &Tokens);
1233 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1235 SmallVectorImpl<AsmToken> &Tokens);
1236 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1237 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1238 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1241 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1242 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1243 void initializeGprCountSymbol(RegisterKind RegKind);
1244 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1250 OperandMode_Default,
1254 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1256 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1257 const MCInstrInfo &MII)
1258 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1259 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1264 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1266 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1267 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1268 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1269 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1271 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1272 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1273 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1275 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1276 initializeGprCountSymbol(IS_VGPR);
1277 initializeGprCountSymbol(IS_SGPR);
1282 createConstantSymbol(Symbol, Code);
1284 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1285 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1286 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1326 bool hasBVHRayTracingInsts()
const {
1327 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1330 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1332 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1334 bool hasInv2PiInlineImm()
const {
1335 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1338 bool has64BitLiterals()
const {
1339 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1342 bool hasFlatOffsets()
const {
1343 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1346 bool hasTrue16Insts()
const {
1347 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1351 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1354 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1356 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1358 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1360 bool hasPartialNSAEncoding()
const {
1361 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1364 bool hasGloballyAddressableScratch()
const {
1365 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1368 bool hasCacheFillSize()
const {
1369 return getFeatureBits()[AMDGPU::FeatureCacheFillSize];
1382 AMDGPUTargetStreamer &getTargetStreamer() {
1383 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1384 return static_cast<AMDGPUTargetStreamer &
>(TS);
1390 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1393 const MCRegisterInfo *getMRI()
const {
1397 const MCInstrInfo *getMII()
const {
return &MII; }
1401 int16_t getTargetRegClass(
unsigned TargetRCIdx)
const {
1407 const FeatureBitset &getFeatureBits()
const {
1408 return getSTI().getFeatureBits();
1411 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1412 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1413 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1415 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1416 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1417 bool isForcedDPP()
const {
return ForcedDPP; }
1418 bool isForcedSDWA()
const {
return ForcedSDWA; }
1419 ArrayRef<unsigned> getMatchedVariants()
const;
1420 StringRef getMatchedVariantName()
const;
1422 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1423 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1424 bool RestoreOnFailure);
1425 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1426 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1427 SMLoc &EndLoc)
override;
1428 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1429 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1430 unsigned Kind)
override;
1431 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1434 bool MatchingInlineAsm)
override;
1435 bool ParseDirective(AsmToken DirectiveID)
override;
1436 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1437 void onEndOfFile()
override;
1439 OperandMode
Mode = OperandMode_Default);
1440 StringRef parseMnemonicSuffix(StringRef Name);
1441 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1447 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1451 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1452 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1454 ParseStatus parseOperandArrayWithPrefix(
1456 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1457 bool (*ConvertResult)(int64_t &) =
nullptr);
1461 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1462 bool IgnoreNegative =
false);
1463 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1468 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1472 ArrayRef<const char *> Ids,
1476 ArrayRef<const char *> Ids,
1477 AMDGPUOperand::ImmTy
Type);
1480 bool isOperandModifier(
const AsmToken &Token,
1481 const AsmToken &NextToken)
const;
1482 bool isRegOrOperandModifier(
const AsmToken &Token,
1483 const AsmToken &NextToken)
const;
1484 bool isNamedOperandModifier(
const AsmToken &Token,
1485 const AsmToken &NextToken)
const;
1486 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1487 const AsmToken &NextToken)
const;
1488 bool parseSP3NegModifier();
1495 bool AllowImm =
true);
1497 bool AllowImm =
true);
1503 AMDGPUOperand::ImmTy ImmTy);
1508 AMDGPUOperand::ImmTy
Type);
1512 AMDGPUOperand::ImmTy
Type);
1516 AMDGPUOperand::ImmTy
Type);
1520 ParseStatus parseDfmtNfmt(int64_t &
Format);
1521 ParseStatus parseUfmt(int64_t &
Format);
1522 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1524 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1527 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1528 ParseStatus parseNumericFormat(int64_t &
Format);
1532 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1533 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1538 bool parseCnt(int64_t &IntVal);
1541 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1542 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1545 bool parseDelay(int64_t &Delay);
1551 struct OperandInfoTy {
1554 bool IsSymbolic =
false;
1555 bool IsDefined =
false;
1557 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1560 struct StructuredOpField : OperandInfoTy {
1564 bool IsDefined =
false;
1566 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1567 unsigned Width, int64_t
Default)
1568 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1569 virtual ~StructuredOpField() =
default;
1571 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1572 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1576 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1578 return Error(Parser,
"not supported on this GPU");
1580 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1588 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1589 OperandInfoTy &Stream);
1590 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1591 const OperandInfoTy &Stream);
1593 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1594 OperandInfoTy &Width);
1599 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1606 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1608 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1612 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1617 bool validateBF16InlineConst(
const MCInst &Inst,
1620 bool validateConstantBusLimitations(
const MCInst &Inst,
1622 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1625 bool tryVOPD(
const MCInst &Inst);
1626 bool tryVOPD3(
const MCInst &Inst);
1627 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1629 bool validateIntClampSupported(
const MCInst &Inst);
1630 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1631 bool validateMIMGGatherDMask(
const MCInst &Inst);
1633 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1634 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1635 bool validateMIMGD16(
const MCInst &Inst);
1637 bool validateTensorR128(
const MCInst &Inst);
1638 bool validateMIMGMSAA(
const MCInst &Inst);
1639 bool validateOpSel(
const MCInst &Inst);
1640 bool validateTrue16OpSel(
const MCInst &Inst);
1641 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1643 bool validateVccOperand(MCRegister
Reg)
const;
1648 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1649 bool validateVGPRAlign(
const MCInst &Inst)
const;
1653 bool validateDivScale(
const MCInst &Inst);
1658 const unsigned CPol);
1664 bool validateClusterBarrierIsFirst(
const MCInst &Inst,
1667 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1668 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1669 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1670 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1672 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1673 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1674 ArrayRef<unsigned> Variants);
1675 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1677 bool isId(
const StringRef Id)
const;
1678 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1680 StringRef getId()
const;
1681 bool trySkipId(
const StringRef Id);
1682 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1686 bool parseString(StringRef &Val,
1687 const StringRef ErrMsg =
"expected a string");
1688 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1694 StringRef getTokenStr()
const;
1695 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1697 SMLoc getLoc()
const;
1701 void onBeginOfFile()
override;
1706 void emitTargetDirective();
1707 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1719 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1720 const unsigned MaxVal,
const Twine &ErrMsg,
1722 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1723 const unsigned MinVal,
const unsigned MaxVal,
1724 const StringRef ErrMsg);
1726 bool parseSwizzleOffset(int64_t &
Imm);
1727 bool parseSwizzleMacro(int64_t &
Imm);
1728 bool parseSwizzleQuadPerm(int64_t &
Imm);
1729 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1730 bool parseSwizzleBroadcast(int64_t &
Imm);
1731 bool parseSwizzleSwap(int64_t &
Imm);
1732 bool parseSwizzleReverse(int64_t &
Imm);
1733 bool parseSwizzleFFT(int64_t &
Imm);
1734 bool parseSwizzleRotate(int64_t &
Imm);
1737 int64_t parseGPRIdxMacro();
1740 cvtMubufImpl(Inst,
Operands,
false);
1743 cvtMubufImpl(Inst,
Operands,
true);
1749 OptionalImmIndexMap &OptionalIdx);
1758 OptionalImmIndexMap &OptionalIdx);
1760 OptionalImmIndexMap &OptionalIdx);
1764 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1766 bool parseDimId(
unsigned &Encoding);
1768 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1772 int64_t parseDPPCtrlSel(StringRef Ctrl);
1773 int64_t parseDPPCtrlPerm();
1779 bool IsDPP8 =
false);
1785 AMDGPUOperand::ImmTy
Type);
1793 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1796 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1797 bool SkipSrcVcc =
false);
1907bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
1917 if (!isImmTy(ImmTyNone)) {
1922 if (getModifiers().
Lit != LitModifier::None)
1932 if (type == MVT::f64 || type == MVT::i64) {
1934 AsmParser->hasInv2PiInlineImm());
1937 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
1956 APFloat::rmNearestTiesToEven, &Lost);
1963 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1965 AsmParser->hasInv2PiInlineImm());
1970 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
1971 AsmParser->hasInv2PiInlineImm());
1975 if (type == MVT::f64 || type == MVT::i64) {
1977 AsmParser->hasInv2PiInlineImm());
1986 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
1987 AsmParser->hasInv2PiInlineImm());
1991 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
1992 AsmParser->hasInv2PiInlineImm());
1995bool AMDGPUOperand::isLiteralImm(MVT type)
const {
1997 if (!isImmTy(ImmTyNone)) {
2002 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2007 if (type == MVT::f64 && hasFPModifiers()) {
2027 if (type == MVT::f64) {
2032 if (type == MVT::i64) {
2045 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2046 : (type == MVT::v2i16) ? MVT::f32
2047 : (type == MVT::v2f32) ? MVT::f32
2050 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2054bool AMDGPUOperand::isRegClassTarget(
unsigned TargetRCIdx)
const {
2057 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2058 return RCID >= 0 && isRegClass(RCID);
2061bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2062 return isRegKind() &&
2063 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2066bool AMDGPUOperand::isVRegWithInputMods()
const {
2067 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2069 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2070 isRegClassTarget(AMDGPU::VReg_64_AlignTarget));
2073template <
bool IsFake16>
2074bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2075 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2076 : AMDGPU::VGPR_16_Lo128RegClassID);
2079template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2080 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2081 : AMDGPU::VGPR_16RegClassID);
2084bool AMDGPUOperand::isT16_LO16VRegWithInputMods()
const {
2085 return isRegClass(AMDGPU::VGPR_16_LO16RegClassID);
2088bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2089 if (AsmParser->isVI())
2091 if (AsmParser->isGFX9Plus())
2092 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2096bool AMDGPUOperand::isSDWAFP16Operand()
const {
2097 return isSDWAOperand(MVT::f16);
2100bool AMDGPUOperand::isSDWAFP32Operand()
const {
2101 return isSDWAOperand(MVT::f32);
2104bool AMDGPUOperand::isSDWAInt16Operand()
const {
2105 return isSDWAOperand(MVT::i16);
2108bool AMDGPUOperand::isSDWAInt32Operand()
const {
2109 return isSDWAOperand(MVT::i32);
2112bool AMDGPUOperand::isBoolReg()
const {
2113 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2114 (AsmParser->isWave32() && isSCSrc_b32()));
2118 unsigned Size)
const {
2119 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2134void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2135 bool ApplyModifiers)
const {
2145 addLiteralImmOperand(Inst,
Imm.Val,
2146 ApplyModifiers & isImmTy(ImmTyNone) &&
2147 Imm.Mods.hasFPModifiers());
2149 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2154void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2155 bool ApplyModifiers)
const {
2156 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2161 if (ApplyModifiers) {
2163 const unsigned Size =
2165 Val = applyInputFPModifiers(Val,
Size);
2169 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2171 bool CanUse64BitLiterals =
2174 MCContext &Ctx = AsmParser->getContext();
2185 if (
Lit == LitModifier::None &&
2187 AsmParser->hasInv2PiInlineImm())) {
2195 bool HasMandatoryLiteral =
2198 if (
Literal.getLoBits(32) != 0 &&
2199 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2200 !HasMandatoryLiteral) {
2201 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2203 "Can't encode literal as exact 64-bit floating-point operand. "
2204 "Low 32-bits will be set to zero");
2205 Val &= 0xffffffff00000000u;
2211 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2217 Lit = LitModifier::Lit64;
2218 }
else if (
Lit == LitModifier::Lit) {
2232 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2234 Lit = LitModifier::Lit64;
2241 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2242 Literal == 0x3fc45f306725feed) {
2282 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2289 if (
Lit != LitModifier::None) {
2320 if (
Lit == LitModifier::None &&
2330 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2338 if (
Lit == LitModifier::None &&
2346 if (!AsmParser->has64BitLiterals()) {
2347 Val =
static_cast<uint64_t>(Val) << 32;
2354 if (
Lit == LitModifier::Lit ||
2356 Val =
static_cast<uint64_t>(Val) << 32;
2360 if (
Lit == LitModifier::Lit)
2387 if (
Lit != LitModifier::None) {
2395void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2401bool AMDGPUOperand::isInlineValue()
const {
2409void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2420 if (Is == IS_VGPR) {
2425 return AMDGPU::VGPR_32RegClassID;
2427 return AMDGPU::VReg_64RegClassID;
2429 return AMDGPU::VReg_96RegClassID;
2431 return AMDGPU::VReg_128RegClassID;
2433 return AMDGPU::VReg_160RegClassID;
2435 return AMDGPU::VReg_192RegClassID;
2437 return AMDGPU::VReg_224RegClassID;
2439 return AMDGPU::VReg_256RegClassID;
2441 return AMDGPU::VReg_288RegClassID;
2443 return AMDGPU::VReg_320RegClassID;
2445 return AMDGPU::VReg_352RegClassID;
2447 return AMDGPU::VReg_384RegClassID;
2449 return AMDGPU::VReg_512RegClassID;
2451 return AMDGPU::VReg_1024RegClassID;
2453 }
else if (Is == IS_TTMP) {
2458 return AMDGPU::TTMP_32RegClassID;
2460 return AMDGPU::TTMP_64RegClassID;
2462 return AMDGPU::TTMP_128RegClassID;
2464 return AMDGPU::TTMP_256RegClassID;
2466 return AMDGPU::TTMP_512RegClassID;
2468 }
else if (Is == IS_SGPR) {
2473 return AMDGPU::SGPR_32RegClassID;
2475 return AMDGPU::SGPR_64RegClassID;
2477 return AMDGPU::SGPR_96RegClassID;
2479 return AMDGPU::SGPR_128RegClassID;
2481 return AMDGPU::SGPR_160RegClassID;
2483 return AMDGPU::SGPR_192RegClassID;
2485 return AMDGPU::SGPR_224RegClassID;
2487 return AMDGPU::SGPR_256RegClassID;
2489 return AMDGPU::SGPR_288RegClassID;
2491 return AMDGPU::SGPR_320RegClassID;
2493 return AMDGPU::SGPR_352RegClassID;
2495 return AMDGPU::SGPR_384RegClassID;
2497 return AMDGPU::SGPR_512RegClassID;
2499 }
else if (Is == IS_AGPR) {
2504 return AMDGPU::AGPR_32RegClassID;
2506 return AMDGPU::AReg_64RegClassID;
2508 return AMDGPU::AReg_96RegClassID;
2510 return AMDGPU::AReg_128RegClassID;
2512 return AMDGPU::AReg_160RegClassID;
2514 return AMDGPU::AReg_192RegClassID;
2516 return AMDGPU::AReg_224RegClassID;
2518 return AMDGPU::AReg_256RegClassID;
2520 return AMDGPU::AReg_288RegClassID;
2522 return AMDGPU::AReg_320RegClassID;
2524 return AMDGPU::AReg_352RegClassID;
2526 return AMDGPU::AReg_384RegClassID;
2528 return AMDGPU::AReg_512RegClassID;
2530 return AMDGPU::AReg_1024RegClassID;
2538 .
Case(
"exec", AMDGPU::EXEC)
2539 .
Case(
"vcc", AMDGPU::VCC)
2540 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2541 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2542 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2543 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2544 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2545 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2546 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2547 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2548 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2549 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2550 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2551 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2552 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2553 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2554 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2555 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2556 .
Case(
"m0", AMDGPU::M0)
2557 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2558 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2559 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2560 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2561 .
Case(
"scc", AMDGPU::SRC_SCC)
2562 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2563 .
Case(
"tba", AMDGPU::TBA)
2564 .
Case(
"tma", AMDGPU::TMA)
2565 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2566 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2567 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2568 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2569 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2570 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2571 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2572 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2573 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2574 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2575 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2576 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2577 .
Case(
"pc", AMDGPU::PC_REG)
2578 .
Case(
"null", AMDGPU::SGPR_NULL)
2582bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2583 SMLoc &EndLoc,
bool RestoreOnFailure) {
2584 auto R = parseRegister();
2588 RegNo =
R->getReg();
2589 StartLoc =
R->getStartLoc();
2590 EndLoc =
R->getEndLoc();
2594bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2596 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2599ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2601 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2602 bool PendingErrors = getParser().hasPendingError();
2603 getParser().clearPendingErrors();
2611bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2612 RegisterKind RegKind,
2614 RegisterKind RegKind1, SMLoc Loc) {
2616 if (RegKind == IS_SGPR) {
2617 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2618 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2619 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2625 if (RegKind != RegKind1) {
2626 Error(Loc,
"registers in a list must be of the same kind");
2632 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2637 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2638 Reg = AMDGPU::FLAT_SCR;
2642 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2643 Reg = AMDGPU::XNACK_MASK;
2647 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2652 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2657 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2662 Error(Loc,
"register does not fit in the list");
2668 if (Reg1 !=
Reg + RegWidth / 32) {
2669 Error(Loc,
"registers in a list must have consecutive indices");
2685 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2686 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2690 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2696 if (Str.starts_with(
Reg.Name))
2702 return !Str.getAsInteger(10, Num);
2705bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2706 const AsmToken &NextToken)
const {
2721 StringRef RegSuffix = Str.substr(
RegName.size());
2722 if (!RegSuffix.
empty()) {
2739bool AMDGPUAsmParser::isRegister() {
2740 return isRegister(
getToken(), peekToken());
2743MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2744 unsigned SubReg,
unsigned RegWidth,
2748 unsigned AlignSize = 1;
2749 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2755 if (RegNum % AlignSize != 0) {
2756 Error(Loc,
"invalid register alignment");
2757 return MCRegister();
2760 unsigned RegIdx = RegNum / AlignSize;
2763 Error(Loc,
"invalid or unsupported register size");
2764 return MCRegister();
2768 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2769 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2770 Error(Loc,
"register index is out of range");
2771 return AMDGPU::NoRegister;
2774 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2775 Error(Loc,
"register index is out of range");
2776 return MCRegister();
2785 Error(Loc,
"invalid subregister");
2791bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2793 int64_t RegLo, RegHi;
2797 SMLoc FirstIdxLoc = getLoc();
2804 SecondIdxLoc = getLoc();
2815 Error(FirstIdxLoc,
"invalid register index");
2820 Error(SecondIdxLoc,
"invalid register index");
2824 if (RegLo > RegHi) {
2825 Error(FirstIdxLoc,
"first register index should not exceed second index");
2829 if (RegHi == RegLo) {
2830 StringRef RegSuffix = getTokenStr();
2831 if (RegSuffix ==
".l") {
2832 SubReg = AMDGPU::lo16;
2834 }
else if (RegSuffix ==
".h") {
2835 SubReg = AMDGPU::hi16;
2840 Num =
static_cast<unsigned>(RegLo);
2841 RegWidth = 32 * ((RegHi - RegLo) + 1);
2846MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2849 SmallVectorImpl<AsmToken> &Tokens) {
2855 RegKind = IS_SPECIAL;
2862MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2865 SmallVectorImpl<AsmToken> &Tokens) {
2867 StringRef
RegName = getTokenStr();
2868 auto Loc = getLoc();
2872 Error(Loc,
"invalid register name");
2873 return MCRegister();
2881 unsigned SubReg = NoSubRegister;
2882 bool IsRange =
false;
2883 if (!RegSuffix.
empty()) {
2885 SubReg = AMDGPU::lo16;
2887 SubReg = AMDGPU::hi16;
2891 Error(Loc,
"invalid register index");
2892 return MCRegister();
2898 if (!ParseRegRange(RegNum, RegWidth, SubReg))
2899 return MCRegister();
2903 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2904 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
2905 if (RegKind == IS_SGPR && IsRange
2906 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
2907 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
2908 Error(Loc,
"register index is out of range");
2909 return MCRegister();
2915MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
2916 unsigned &RegNum,
unsigned &RegWidth,
2917 SmallVectorImpl<AsmToken> &Tokens) {
2919 auto ListLoc = getLoc();
2922 "expected a register or a list of registers")) {
2923 return MCRegister();
2928 auto Loc = getLoc();
2929 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
2930 return MCRegister();
2931 if (RegWidth != 32) {
2932 Error(Loc,
"expected a single 32-bit register");
2933 return MCRegister();
2937 RegisterKind NextRegKind;
2939 unsigned NextRegNum, NextRegWidth;
2942 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
2944 return MCRegister();
2946 if (NextRegWidth != 32) {
2947 Error(Loc,
"expected a single 32-bit register");
2948 return MCRegister();
2950 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
2952 return MCRegister();
2956 "expected a comma or a closing square bracket")) {
2957 return MCRegister();
2961 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
2966bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
2967 MCRegister &
Reg,
unsigned &RegNum,
2969 SmallVectorImpl<AsmToken> &Tokens) {
2970 auto Loc = getLoc();
2974 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2976 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2978 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2983 assert(Parser.hasPendingError());
2987 if (!subtargetHasRegister(*
TRI,
Reg)) {
2988 if (
Reg == AMDGPU::SGPR_NULL) {
2989 Error(Loc,
"'null' operand is not supported on this GPU");
2992 " register not available on this GPU");
3000bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3001 MCRegister &
Reg,
unsigned &RegNum,
3003 bool RestoreOnFailure ) {
3007 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3008 if (RestoreOnFailure) {
3009 while (!Tokens.
empty()) {
3018std::optional<StringRef>
3019AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3022 return StringRef(
".amdgcn.next_free_vgpr");
3024 return StringRef(
".amdgcn.next_free_sgpr");
3026 return std::nullopt;
3030void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3031 auto SymbolName = getGprCountSymbolName(RegKind);
3032 assert(SymbolName &&
"initializing invalid register kind");
3038bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3039 unsigned DwordRegIndex,
3040 unsigned RegWidth) {
3045 auto SymbolName = getGprCountSymbolName(RegKind);
3050 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3054 return !
Error(getLoc(),
3055 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3059 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3061 if (OldCount <= NewMax)
3067std::unique_ptr<AMDGPUOperand>
3068AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3070 SMLoc StartLoc = Tok.getLoc();
3071 SMLoc EndLoc = Tok.getEndLoc();
3072 RegisterKind RegKind;
3074 unsigned RegNum, RegWidth;
3076 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3080 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3083 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3084 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3091 if (isRegister() || isModifier())
3094 if (
Lit == LitModifier::None) {
3095 if (trySkipId(
"lit"))
3096 Lit = LitModifier::Lit;
3097 else if (trySkipId(
"lit64"))
3098 Lit = LitModifier::Lit64;
3100 if (
Lit != LitModifier::None) {
3103 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3112 const auto &NextTok = peekToken();
3115 bool Negate =
false;
3123 AMDGPUOperand::Modifiers Mods;
3131 StringRef Num = getTokenStr();
3134 APFloat RealVal(APFloat::IEEEdouble());
3135 auto roundMode = APFloat::rmNearestTiesToEven;
3136 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3139 RealVal.changeSign();
3142 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3143 S, AMDGPUOperand::ImmTyNone,
true));
3144 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3145 Op.setModifiers(Mods);
3154 if (HasSP3AbsModifier) {
3163 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3166 if (Parser.parseExpression(Expr))
3170 if (Expr->evaluateAsAbsolute(IntVal)) {
3172 return Error(S,
"literal value out of range");
3173 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3174 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3175 Op.setModifiers(Mods);
3177 if (
Lit != LitModifier::None)
3179 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3192 if (
auto R = parseRegister()) {
3202 ParseStatus Res = parseReg(
Operands);
3210bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3211 const AsmToken &NextToken)
const {
3214 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3219bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3220 const AsmToken &NextToken)
const {
3224bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3225 const AsmToken &NextToken)
const {
3226 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3229bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3230 const AsmToken &NextToken)
const {
3231 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3248bool AMDGPUAsmParser::isModifier() {
3251 AsmToken NextToken[2];
3252 peekTokens(NextToken);
3256 bool IsOpcodeModifier = isOpcodeModifierWithVal(Tok, NextToken[0]) &&
3259 return isOperandModifier(Tok, NextToken[0]) ||
3261 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3287bool AMDGPUAsmParser::parseSP3NegModifier() {
3289 AsmToken NextToken[2];
3290 peekTokens(NextToken);
3293 (isRegister(NextToken[0], NextToken[1]) ||
3311 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3313 SP3Neg = parseSP3NegModifier();
3316 Neg = trySkipId(
"neg");
3318 return Error(Loc,
"expected register or immediate");
3322 Abs = trySkipId(
"abs");
3327 if (trySkipId(
"lit")) {
3328 Lit = LitModifier::Lit;
3331 }
else if (trySkipId(
"lit64")) {
3332 Lit = LitModifier::Lit64;
3335 if (!has64BitLiterals())
3336 return Error(Loc,
"lit64 is not supported on this GPU");
3342 return Error(Loc,
"expected register or immediate");
3351 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3355 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3356 Error(Loc,
"expected immediate with lit modifier");
3358 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3364 if (
Lit != LitModifier::None &&
3368 AMDGPUOperand::Modifiers Mods;
3369 Mods.Abs = Abs || SP3Abs;
3370 Mods.Neg = Neg || SP3Neg;
3373 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3374 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3376 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3377 Op.setModifiers(Mods);
3385 bool Sext = trySkipId(
"sext");
3386 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3401 AMDGPUOperand::Modifiers Mods;
3404 if (Mods.hasIntModifiers()) {
3405 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3407 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3408 Op.setModifiers(Mods);
3415 return parseRegOrImmWithFPInputMods(
Operands,
false);
3419 return parseRegOrImmWithIntInputMods(
Operands,
false);
3426 if (!trySkipId(
"rsrcidx"))
3432 SMLoc RegLoc = getLoc();
3433 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3441 if (!
Reg->isRsrcReg32())
3442 return Error(RegLoc,
"rsrcidx operand must be a 32-bit SGPR or VGPR");
3452 auto Loc = getLoc();
3453 if (trySkipId(
"off")) {
3455 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3462 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3471unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3476 return Match_InvalidOperand;
3478 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3479 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3482 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3484 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3485 return Match_InvalidOperand;
3493 if (tryAnotherVOPDEncoding(Inst))
3494 return Match_InvalidOperand;
3496 return Match_Success;
3500 static const unsigned Variants[] = {
3509ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3510 if (isForcedDPP() && isForcedVOP3()) {
3514 if (getForcedEncodingSize() == 32) {
3519 if (isForcedVOP3()) {
3524 if (isForcedSDWA()) {
3530 if (isForcedDPP()) {
3538StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3539 if (isForcedDPP() && isForcedVOP3())
3542 if (getForcedEncodingSize() == 32)
3558AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3562 case AMDGPU::FLAT_SCR:
3564 case AMDGPU::VCC_LO:
3565 case AMDGPU::VCC_HI:
3572 return MCRegister();
3579bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3580 unsigned OpIdx)
const {
3588 const MCOperand &MO = Inst.
getOperand(OpIdx);
3638unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3644 case AMDGPU::V_LSHLREV_B64_e64:
3645 case AMDGPU::V_LSHLREV_B64_gfx10:
3646 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3647 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3648 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3649 case AMDGPU::V_LSHRREV_B64_e64:
3650 case AMDGPU::V_LSHRREV_B64_gfx10:
3651 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3652 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3653 case AMDGPU::V_ASHRREV_I64_e64:
3654 case AMDGPU::V_ASHRREV_I64_gfx10:
3655 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3656 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3657 case AMDGPU::V_LSHL_B64_e64:
3658 case AMDGPU::V_LSHR_B64_e64:
3659 case AMDGPU::V_ASHR_I64_e64:
3672 bool AddMandatoryLiterals =
false) {
3675 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3679 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3681 return {getNamedOperandIdx(Opcode, OpName::src0X),
3682 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3683 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3684 getNamedOperandIdx(Opcode, OpName::src0Y),
3685 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3686 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3691 return {getNamedOperandIdx(Opcode, OpName::src0),
3692 getNamedOperandIdx(Opcode, OpName::src1),
3693 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3696bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3697 const MCOperand &MO = Inst.
getOperand(OpIdx);
3699 return !isInlineConstant(Inst, OpIdx);
3706 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3717 const unsigned Opcode = Inst.
getOpcode();
3718 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3721 if (!LaneSelOp.
isReg())
3724 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3727bool AMDGPUAsmParser::validateConstantBusLimitations(
3729 const unsigned Opcode = Inst.
getOpcode();
3730 const MCInstrDesc &
Desc = MII.
get(Opcode);
3731 MCRegister LastSGPR;
3732 unsigned ConstantBusUseCount = 0;
3733 unsigned NumLiterals = 0;
3734 unsigned LiteralSize;
3750 SmallDenseSet<MCRegister> SGPRsUsed;
3751 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3753 SGPRsUsed.
insert(SGPRUsed);
3754 ++ConstantBusUseCount;
3759 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3761 for (
int OpIdx : OpIndices) {
3765 const MCOperand &MO = Inst.
getOperand(OpIdx);
3766 if (usesConstantBus(Inst, OpIdx)) {
3775 if (SGPRsUsed.
insert(LastSGPR).second) {
3776 ++ConstantBusUseCount;
3796 if (NumLiterals == 0) {
3799 }
else if (LiteralSize !=
Size) {
3805 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3807 "invalid operand (violates constant bus restrictions)");
3814std::optional<unsigned>
3815AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3817 const unsigned Opcode = Inst.
getOpcode();
3823 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3824 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3833 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3834 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3835 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3836 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3837 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3838 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3842 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3843 int I = getNamedOperandIdx(Opcode, OpName);
3847 int64_t
Imm =
Op.getImm();
3853 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
3854 OpName::vsrc2Y, OpName::imm}) {
3855 int I = getNamedOperandIdx(Opcode, OpName);
3865 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
3866 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
3868 return InvalidCompOprIdx;
3871bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
3878 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
3879 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
3880 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
3882 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
3886 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
3887 if (!InvalidCompOprIdx.has_value())
3890 auto CompOprIdx = *InvalidCompOprIdx;
3893 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
3894 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3897 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
3898 if (CompOprIdx == VOPD::Component::DST) {
3900 Error(Loc,
"dst registers must be distinct");
3902 Error(Loc,
"one dst register must be even and the other odd");
3904 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3905 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
3906 " operands must use different VGPR banks");
3914bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
3916 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
3917 if (!InvalidCompOprIdx.has_value())
3921 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
3922 if (InvalidCompOprIdx.has_value()) {
3927 if (*InvalidCompOprIdx == VOPD::Component::DST)
3940bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
3941 const unsigned Opcode = Inst.
getOpcode();
3956 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
3957 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
3958 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
3959 int I = getNamedOperandIdx(Opcode, OpName);
3966 return !tryVOPD3(Inst);
3971bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
3976 return tryVOPD(Inst);
3977 return tryVOPD3(Inst);
3980bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
3985 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
3993bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4001 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4002 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4003 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4008 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4009 hasBVHRayTracingInsts())
4012 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4013 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4018 bool IsPackedD16 =
false;
4021 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4022 IsPackedD16 = D16Idx >= 0;
4024 DataSize = (DataSize + 1) / 2;
4027 if ((VDataSize / 4) == DataSize + TFESize)
4032 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4034 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4036 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4040bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4049 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4051 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4052 AMDGPU::OpName RSrcOpName =
4054 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4055 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4056 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4060 assert(SrsrcIdx > VAddr0Idx);
4063 if (BaseOpcode->
BVH) {
4064 if (IsA16 == BaseOpcode->
A16)
4066 Error(IDLoc,
"image address size does not match a16");
4072 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4073 unsigned ActualAddrSize =
4074 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4076 unsigned ExpectedAddrSize =
4080 if (hasPartialNSAEncoding() &&
4082 int VAddrLastIdx = SrsrcIdx - 1;
4083 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4085 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4088 if (ExpectedAddrSize > 12)
4089 ExpectedAddrSize = 16;
4094 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4098 if (ActualAddrSize == ExpectedAddrSize)
4101 Error(IDLoc,
"image address size does not match dim and a16");
4105bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4112 if (!
Desc.mayLoad() || !
Desc.mayStore())
4115 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4122 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4125bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4132 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4140 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4143bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4157 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4158 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4165bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4172 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4175 if (!BaseOpcode->
MSAA)
4178 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4184 return DimInfo->
MSAA;
4189 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4190 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4191 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4201bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4209 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4220 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4224bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4229 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4232 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4243 "source operand must be either a VGPR or an inline constant");
4250bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4255 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4258 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4262 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4264 "inline constants are not allowed for this operand");
4271bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4279 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4280 if (BlgpIdx != -1) {
4281 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4282 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4292 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4294 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4299 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4301 "wrong register tuple size for blgp value " + Twine(BLGP));
4309 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4319 if (Src2Reg == DstReg)
4324 .getSizeInBits() <= 128)
4327 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4329 "source 2 operand must not partially overlap with dst");
4336bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4340 case V_DIV_SCALE_F32_gfx6_gfx7:
4341 case V_DIV_SCALE_F32_vi:
4342 case V_DIV_SCALE_F32_gfx10:
4343 case V_DIV_SCALE_F64_gfx6_gfx7:
4344 case V_DIV_SCALE_F64_vi:
4345 case V_DIV_SCALE_F64_gfx10:
4352 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4353 AMDGPU::OpName::src2_modifiers}) {
4364bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4371 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4380bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4386 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4393 case AMDGPU::V_SUBREV_F32_e32:
4394 case AMDGPU::V_SUBREV_F32_e64:
4395 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4396 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4397 case AMDGPU::V_SUBREV_F32_e32_vi:
4398 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4399 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4400 case AMDGPU::V_SUBREV_F32_e64_vi:
4402 case AMDGPU::V_SUBREV_CO_U32_e32:
4403 case AMDGPU::V_SUBREV_CO_U32_e64:
4404 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4405 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4407 case AMDGPU::V_SUBBREV_U32_e32:
4408 case AMDGPU::V_SUBBREV_U32_e64:
4409 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4410 case AMDGPU::V_SUBBREV_U32_e32_vi:
4411 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4412 case AMDGPU::V_SUBBREV_U32_e64_vi:
4414 case AMDGPU::V_SUBREV_U32_e32:
4415 case AMDGPU::V_SUBREV_U32_e64:
4416 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4417 case AMDGPU::V_SUBREV_U32_e32_vi:
4418 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4419 case AMDGPU::V_SUBREV_U32_e64_vi:
4421 case AMDGPU::V_SUBREV_F16_e32:
4422 case AMDGPU::V_SUBREV_F16_e64:
4423 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4424 case AMDGPU::V_SUBREV_F16_e32_vi:
4425 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4426 case AMDGPU::V_SUBREV_F16_e64_vi:
4428 case AMDGPU::V_SUBREV_U16_e32:
4429 case AMDGPU::V_SUBREV_U16_e64:
4430 case AMDGPU::V_SUBREV_U16_e32_vi:
4431 case AMDGPU::V_SUBREV_U16_e64_vi:
4433 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4434 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4435 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4437 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4438 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4440 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4441 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4443 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4444 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4446 case AMDGPU::V_LSHRREV_B32_e32:
4447 case AMDGPU::V_LSHRREV_B32_e64:
4448 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4449 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4450 case AMDGPU::V_LSHRREV_B32_e32_vi:
4451 case AMDGPU::V_LSHRREV_B32_e64_vi:
4452 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4453 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4455 case AMDGPU::V_ASHRREV_I32_e32:
4456 case AMDGPU::V_ASHRREV_I32_e64:
4457 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4458 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4459 case AMDGPU::V_ASHRREV_I32_e32_vi:
4460 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4461 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4462 case AMDGPU::V_ASHRREV_I32_e64_vi:
4464 case AMDGPU::V_LSHLREV_B32_e32:
4465 case AMDGPU::V_LSHLREV_B32_e64:
4466 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4467 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4468 case AMDGPU::V_LSHLREV_B32_e32_vi:
4469 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4470 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4471 case AMDGPU::V_LSHLREV_B32_e64_vi:
4473 case AMDGPU::V_LSHLREV_B16_e32:
4474 case AMDGPU::V_LSHLREV_B16_e64:
4475 case AMDGPU::V_LSHLREV_B16_e32_vi:
4476 case AMDGPU::V_LSHLREV_B16_e64_vi:
4477 case AMDGPU::V_LSHLREV_B16_gfx10:
4479 case AMDGPU::V_LSHRREV_B16_e32:
4480 case AMDGPU::V_LSHRREV_B16_e64:
4481 case AMDGPU::V_LSHRREV_B16_e32_vi:
4482 case AMDGPU::V_LSHRREV_B16_e64_vi:
4483 case AMDGPU::V_LSHRREV_B16_gfx10:
4485 case AMDGPU::V_ASHRREV_I16_e32:
4486 case AMDGPU::V_ASHRREV_I16_e64:
4487 case AMDGPU::V_ASHRREV_I16_e32_vi:
4488 case AMDGPU::V_ASHRREV_I16_e64_vi:
4489 case AMDGPU::V_ASHRREV_I16_gfx10:
4491 case AMDGPU::V_LSHLREV_B64_e64:
4492 case AMDGPU::V_LSHLREV_B64_gfx10:
4493 case AMDGPU::V_LSHLREV_B64_vi:
4495 case AMDGPU::V_LSHRREV_B64_e64:
4496 case AMDGPU::V_LSHRREV_B64_gfx10:
4497 case AMDGPU::V_LSHRREV_B64_vi:
4499 case AMDGPU::V_ASHRREV_I64_e64:
4500 case AMDGPU::V_ASHRREV_I64_gfx10:
4501 case AMDGPU::V_ASHRREV_I64_vi:
4503 case AMDGPU::V_PK_LSHLREV_B16:
4504 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4505 case AMDGPU::V_PK_LSHLREV_B16_vi:
4507 case AMDGPU::V_PK_LSHRREV_B16:
4508 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4509 case AMDGPU::V_PK_LSHRREV_B16_vi:
4510 case AMDGPU::V_PK_ASHRREV_I16:
4511 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4512 case AMDGPU::V_PK_ASHRREV_I16_vi:
4519bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4521 const unsigned Opcode = Inst.
getOpcode();
4530 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4531 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4535 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4539 "lds_direct is not supported on this GPU");
4545 "lds_direct cannot be used with this instruction");
4549 if (SrcName != OpName::src0) {
4551 "lds_direct may be used as src0 only");
4561 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4562 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4563 if (
Op.isFlatOffset())
4564 return Op.getStartLoc();
4569bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4572 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4577 return validateFlatOffset(Inst,
Operands);
4580 return validateSMEMOffset(Inst,
Operands);
4585 const unsigned OffsetSize = 24;
4586 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4588 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4589 "-bit unsigned offset for buffer ops");
4593 const unsigned OffsetSize = 16;
4594 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4596 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4603bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4609 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4613 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4615 "flat offset modifier is not supported on this GPU");
4622 bool AllowNegative =
4624 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4626 Twine(
"expected a ") +
4627 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4628 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4637 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4638 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4639 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4640 return Op.getStartLoc();
4645bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4654 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4670 ?
"expected a 23-bit unsigned offset for buffer ops"
4671 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4672 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4673 :
"expected a 21-bit signed offset");
4683bool AMDGPUAsmParser::validateBF16InlineConst(
const MCInst &Inst,
4685 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4700 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4704 const MCOperandInfo &Src0Info =
Desc.operands()[Src0Idx];
4709 if (!
Src0.isImm() ||
4711 hasInv2PiInlineImm()))
4716 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
4717 if (ModsIdx != -1 &&
4723 "bf16 inline constant is read from the high half of the fp32 inline "
4724 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4728bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4731 const MCInstrDesc &
Desc = MII.
get(Opcode);
4735 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4736 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4738 const int OpIndices[] = {Src0Idx, Src1Idx};
4740 unsigned NumExprs = 0;
4741 unsigned NumLiterals = 0;
4744 for (
int OpIdx : OpIndices) {
4748 const MCOperand &MO = Inst.
getOperand(OpIdx);
4752 std::optional<int64_t>
Imm;
4755 }
else if (MO.
isExpr()) {
4764 if (!
Imm.has_value()) {
4766 }
else if (!isInlineConstant(Inst, OpIdx)) {
4770 if (NumLiterals == 0 || LiteralValue !=
Value) {
4778 if (NumLiterals + NumExprs <= 1)
4782 "only one unique literal operand is allowed");
4786bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4789 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4797 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4798 if (OpSelIdx != -1) {
4802 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4803 if (OpSelHiIdx != -1) {
4812 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4819 if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] &&
4820 (
Opc == AMDGPU::V_PK_ADD_F32_vi ||
Opc == AMDGPU::V_PK_MUL_F32_vi ||
4821 Opc == AMDGPU::V_PK_FMA_F32_vi)) {
4822 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4826 if (
Opc == AMDGPU::V_PK_FMA_F32_vi &&
4827 (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1)))
4830 else if ((OpSel & 3) == 2)
4838 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4839 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4840 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4841 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4850 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4852 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4862 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4863 if (Src2Idx != -1) {
4874bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4875 if (!hasTrue16Insts())
4877 const MCRegisterInfo *MRI = getMRI();
4879 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4885 if (OpSelOpValue == 0)
4887 unsigned OpCount = 0;
4888 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4889 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4890 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4897 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4898 if (OpSelOpIsHi != VGPRSuffixIsHi)
4907bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4908 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4920 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
4931 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4932 AMDGPU::OpName::src1_modifiers,
4933 AMDGPU::OpName::src2_modifiers};
4935 for (
unsigned i = 0; i < 3; ++i) {
4945bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
4948 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
4949 if (DppCtrlIdx >= 0) {
4953 getSTI().
hasFeature(AMDGPU::FeatureDPALU_DPP) &&
4957 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
4958 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
4959 :
"DP ALU dpp only supports row_newbcast");
4964 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
4965 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4968 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4974 "invalid operand for instruction");
4979 "src1 immediate operand invalid for instruction");
4989bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
4990 return (
Reg == AMDGPU::VCC && isWave64()) ||
4991 (
Reg == AMDGPU::VCC_LO && isWave32());
4995bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
4998 const MCInstrDesc &
Desc = MII.
get(Opcode);
4999 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5006 std::optional<unsigned> LiteralOpIdx;
5009 for (
int OpIdx : OpIndices) {
5013 const MCOperand &MO = Inst.
getOperand(OpIdx);
5019 std::optional<int64_t>
Imm;
5025 bool IsAnotherLiteral =
false;
5026 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5027 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5028 if (!
Imm.has_value()) {
5030 IsAnotherLiteral =
true;
5031 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5036 HasMandatoryLiteral);
5048 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5049 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5051 "invalid operand for instruction");
5056 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5057 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5059 "invalid operand for instruction");
5064 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5065 Value =
static_cast<uint32_t
>(
5073 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5074 !getFeatureBits()[FeatureVOP3Literal]) {
5076 "literal operands are not supported");
5080 if (LiteralOpIdx && IsAnotherLiteral) {
5082 getOperandLoc(
Operands, *LiteralOpIdx)),
5083 "only one unique literal operand is allowed");
5087 if (IsAnotherLiteral)
5088 LiteralOpIdx = OpIdx;
5097 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5111bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5117 ? AMDGPU::OpName::data0
5118 : AMDGPU::OpName::vdata;
5120 const MCRegisterInfo *MRI = getMRI();
5121 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5125 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5126 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5130 auto FB = getFeatureBits();
5131 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5132 if (DataAreg < 0 || DstAreg < 0)
5134 return DstAreg == DataAreg;
5137 return DstAreg < 1 && DataAreg < 1;
5140bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5141 auto FB = getFeatureBits();
5142 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5146 const MCRegisterInfo *MRI = getMRI();
5149 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5152 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5156 case AMDGPU::DS_LOAD_TR6_B96:
5157 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5161 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5162 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5166 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5167 if (VAddrIdx != -1) {
5170 if ((
Sub - AMDGPU::VGPR0) & 1)
5175 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5176 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5181 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5182 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5202 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5203 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5205 return Op.getStartLoc();
5210bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5213 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5216 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5219 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5220 auto FB = getFeatureBits();
5221 bool UsesNeg =
false;
5222 if (FB[AMDGPU::FeatureGFX940Insts]) {
5224 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5225 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5226 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5227 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5232 if (IsNeg == UsesNeg)
5235 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5236 :
"invalid modifier: neg is not supported");
5241bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5247 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5248 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5249 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5250 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5253 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5256 if (
Reg == AMDGPU::SGPR_NULL)
5259 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5263bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5268 return validateGWS(Inst,
Operands);
5273 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5278 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5279 Error(S,
"gds modifier is not supported on this GPU");
5287bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5289 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5293 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5294 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5297 const MCRegisterInfo *MRI = getMRI();
5298 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5300 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5303 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5305 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5312bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5316 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5323 if (!hasCacheFillSize()) {
5324 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5327 Error(S,
"Cache fill size is not supported on this GPU");
5328 }
else if (!validateCFSBits(Inst,
Operands)) {
5335 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5338 Error(S,
"scale_offset is not supported on this GPU");
5341 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5344 Error(S,
"nv is not supported on this GPU");
5349 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5352 Error(S,
"scale_offset is not supported for this instruction");
5356 return validateTHAndScopeBits(Inst,
Operands, CPol);
5360 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5361 Error(S,
"cache policy is not supported for SMRD instructions");
5365 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5372 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5376 "scc modifier is not supported for this instruction on this GPU");
5387 :
"instruction must use glc");
5392 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5395 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5397 :
"instruction must not use glc");
5405bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5407 const unsigned CPol) {
5412 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5419 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5424 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5432 return PrintError(
"invalid th value for SMEM instruction");
5439 return PrintError(
"scope and th combination is not valid");
5445 return PrintError(
"invalid th value for atomic instructions");
5448 return PrintError(
"invalid th value for store instructions");
5451 return PrintError(
"invalid th value for load instructions");
5457bool AMDGPUAsmParser::validateCFSBits(
const MCInst &Inst,
5459 const unsigned Opcode = Inst.
getOpcode();
5460 const MCInstrDesc &
Desc = MII.
get(Opcode);
5463 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5471 return PrintError(
"Cache fill size is only supported for VC operations");
5474bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5478 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5480 Error(Loc,
"TFE modifier has no meaning for store instructions");
5488bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5494 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5498 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5501 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5502 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5511 "wrong register tuple size for " +
5516 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5517 !validateFmt(BFmt, AMDGPU::OpName::src1))
5521 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5522 if (AScaleIdx == -1)
5526 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5530 "invalid matrix and scale format combination");
5537bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5540 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5541 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5544 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5547 "sleep forever is unsuported on the target");
5554bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5557 if (
Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5558 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5561 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
5568 "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5572bool AMDGPUAsmParser::validateScaleSel(
const MCInst &Inst,
5575 int ScaleSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::scale_sel);
5576 if (ScaleSelIdx == -1)
5580 case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250:
5581 case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250:
5582 case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250:
5583 case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250:
5584 case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250:
5585 case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250:
5586 case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250:
5587 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250:
5588 case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250:
5591 case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250:
5592 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250:
5593 case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250:
5594 case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250:
5595 case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250:
5596 case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250:
5603 if (getSTI().
hasFeature(AMDGPU::FeatureBlock16ConversionScaleInsts))
5607 if (ScaleSel < MaxSel)
5611 "scale_sel maximum supported value is " + Twine(MaxSel - 1));
5615bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5617 if (!validateLdsDirect(Inst,
Operands))
5619 if (!validateTrue16OpSel(Inst)) {
5621 "op_sel operand conflicts with 16-bit operand suffix");
5624 if (!validateSOPLiteral(Inst,
Operands))
5626 if (!validateVOPLiteral(Inst,
Operands)) {
5629 if (!validateConstantBusLimitations(Inst,
Operands)) {
5632 if (!validateVOPD(Inst,
Operands)) {
5635 if (!validateIntClampSupported(Inst)) {
5637 "integer clamping is not supported on this GPU");
5640 if (!validateOpSel(Inst)) {
5642 "invalid op_sel operand");
5645 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5647 "invalid neg_lo operand");
5650 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5652 "invalid neg_hi operand");
5655 if (!validateDPP(Inst,
Operands)) {
5659 if (!validateMIMGD16(Inst)) {
5661 "d16 modifier is not supported on this GPU");
5664 if (!validateMIMGDim(Inst,
Operands)) {
5665 Error(IDLoc,
"missing dim operand");
5668 if (!validateTensorR128(Inst)) {
5670 "instruction must set modifier r128=0");
5673 if (!validateMIMGMSAA(Inst)) {
5675 "invalid dim; must be MSAA type");
5678 if (!validateMIMGDataSize(Inst, IDLoc)) {
5681 if (!validateMIMGAddrSize(Inst, IDLoc))
5683 if (!validateMIMGAtomicDMask(Inst)) {
5685 "invalid atomic image dmask");
5688 if (!validateMIMGGatherDMask(Inst)) {
5690 "invalid image_gather dmask: only one bit must be set");
5693 if (!validateMovrels(Inst,
Operands)) {
5696 if (!validateOffset(Inst,
Operands)) {
5699 if (!validateBF16InlineConst(Inst,
Operands)) {
5702 if (!validateMAIAccWrite(Inst,
Operands)) {
5705 if (!validateMAISrc2(Inst,
Operands)) {
5708 if (!validateMFMA(Inst,
Operands)) {
5711 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5715 if (!validateAGPRLdSt(Inst)) {
5718 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5719 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5720 :
"invalid register class: agpr loads and stores not supported on "
5724 if (!validateVGPRAlign(Inst)) {
5725 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5732 if (!validateBLGP(Inst,
Operands)) {
5736 if (!validateDivScale(Inst)) {
5737 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5740 if (!validateWaitCnt(Inst,
Operands)) {
5743 if (!validateTFE(Inst,
Operands)) {
5746 if (!validateWMMA(Inst,
Operands)) {
5749 if (!validateMonitorSleep(Inst,
Operands)) {
5752 if (!validateClusterBarrierIsFirst(Inst,
Operands)) {
5755 if (!validateScaleSel(Inst,
Operands)) {
5764 unsigned VariantID = 0);
5768 unsigned VariantID);
5770bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5775bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5776 const FeatureBitset &FBS,
5777 ArrayRef<unsigned> Variants) {
5778 for (
auto Variant : Variants) {
5786bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5788 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5791 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5796 getParser().clearPendingErrors();
5800 StringRef VariantName = getMatchedVariantName();
5801 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5802 return Error(IDLoc, Twine(VariantName,
5803 " variant of this instruction is not supported"));
5807 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5808 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5810 FeatureBitset FeaturesWS32 = getFeatureBits();
5811 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5812 .
flip(AMDGPU::FeatureWavefrontSize32);
5813 FeatureBitset AvailableFeaturesWS32 =
5814 ComputeAvailableFeatures(FeaturesWS32);
5816 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5817 return Error(IDLoc,
"instruction requires wavesize=32");
5821 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5822 return Error(IDLoc,
"instruction not supported on this GPU (" +
5823 getSTI().
getCPU() +
")" +
": " + Mnemo);
5828 return Error(IDLoc,
"invalid instruction" + Suggestion);
5834 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5835 if (
Op.isToken() && InvalidOprIdx > 1) {
5836 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5837 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5842bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5846 bool MatchingInlineAsm) {
5849 unsigned Result = Match_Success;
5854 auto atLeastAsSpecific = [](
unsigned New,
unsigned Cur) {
5855 auto rank = [](
unsigned M) {
5856 return M == Match_MnemonicFail ? 1
5857 :
M == Match_InvalidOperand ? 2
5858 :
M == Match_MissingFeature ? 3
5861 return rank(New) >= rank(Cur);
5864 for (
auto Variant : getMatchedVariants()) {
5867 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5868 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5872 if (R == Match_Success)
5876 if (Result == Match_Success) {
5877 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5880 emitTargetDirective();
5881 Out.emitInstruction(Inst, getSTI());
5888 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5895 case Match_MissingFeature:
5899 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5901 case Match_InvalidOperand: {
5902 SMLoc ErrorLoc = IDLoc;
5903 if (ErrorInfo != ~0ULL) {
5904 if (ErrorInfo >=
Operands.size()) {
5905 return Error(IDLoc,
"too few operands for instruction");
5907 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*
Operands[ErrorInfo];
5908 ErrorLoc = ErrorOp.getStartLoc();
5909 if (ErrorLoc == SMLoc())
5913 return Error(ErrorLoc,
"invalid VOPDY instruction");
5915 return Error(ErrorLoc,
"invalid operand for instruction");
5918 case Match_MnemonicFail:
5924bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5929 if (getParser().parseAbsoluteExpression(Tmp)) {
5932 Ret =
static_cast<uint32_t
>(Tmp);
5936bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5937 if (!getSTI().getTargetTriple().isAMDGCN())
5938 return TokError(
"directive only supported for amdgcn architecture");
5940 std::string TargetIDDirective;
5941 SMLoc TargetStart = getTok().getLoc();
5942 if (getParser().parseEscapedString(TargetIDDirective))
5945 std::optional<AMDGPU::TargetID> MaybeParsed =
5948 return getParser().Error(TargetStart,
5949 "malformed target id '" + TargetIDDirective +
"'");
5952 const Triple &
TT = getSTI().getTargetTriple();
5958 return getParser().Error(
5959 TargetStart,
"target id '" + TargetIDDirective +
5960 "' specifies a processor that is not valid for "
5962 TT.getArchName() +
"'");
5965 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5966 getTargetStreamer().getTargetID();
5969 const Triple &STITriple = getSTI().getTargetTriple();
5970 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5971 return getParser().Error(
5972 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5973 " is incompatible with " +
5974 Twine(CurrentTargetID->toString()));
5978 StringRef DirectiveProcessor =
5981 if (DirectiveISA != ISA) {
5982 return getParser().Error(TargetStart,
5983 ".amdgcn_target directive processor " +
5984 Twine(DirectiveProcessor) +
5985 " does not match the specified processor " +
5986 Twine(getSTI().
getCPU()));
5992 CurrentTargetID->getXnackSetting())) {
5994 ".amdgcn_target directive has conflicting xnack settings");
5998 CurrentTargetID->getSramEccSetting())) {
6000 ".amdgcn_target directive has conflicting sramecc settings");
6006 getTargetStreamer().getTargetID()->setXnackSetting(
6008 getTargetStreamer().getTargetID()->setSramEccSetting(
6014bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
6018bool AMDGPUAsmParser::calculateGPRBlocks(
6019 const FeatureBitset &Features,
const MCExpr *VCCUsed,
6020 const MCExpr *FlatScrUsed,
bool XNACKUsed,
6021 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
6022 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6023 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6028 const MCExpr *
NumSGPRs = NextFreeSGPR;
6029 int64_t EvaluatedSGPRs;
6031 if (
ISA.Major >= 10)
6036 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6037 !Features.
test(FeatureSGPRInitBug) &&
6038 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6039 return OutOfRangeError(SGPRRange);
6041 const MCExpr *ExtraSGPRs =
6045 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6046 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6047 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6048 return OutOfRangeError(SGPRRange);
6050 if (Features.
test(FeatureSGPRInitBug))
6057 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6058 unsigned Granule) ->
const MCExpr * {
6062 const MCExpr *AlignToGPR =
6064 const MCExpr *DivGPR =
6070 VGPRBlocks = GetNumGPRBlocks(
6079bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6080 if (!getSTI().getTargetTriple().isAMDGCN())
6081 return TokError(
"directive only supported for amdgcn architecture");
6084 return TokError(
"directive only supported for amdhsa OS");
6086 StringRef KernelName;
6087 if (getParser().parseIdentifier(KernelName))
6094 AMDGPU::MCKernelDescriptor KD =
6104 const MCExpr *NextFreeVGPR = ZeroExpr;
6106 const MCExpr *NamedBarCnt = ZeroExpr;
6111 const MCExpr *NextFreeSGPR = ZeroExpr;
6114 unsigned ImpliedUserSGPRCount = 0;
6118 std::optional<unsigned> ExplicitUserSGPRCount;
6119 const MCExpr *ReserveVCC = OneExpr;
6120 const MCExpr *ReserveFlatScr = OneExpr;
6121 std::optional<bool> EnableWavefrontSize32;
6128 SMRange IDRange = getTok().getLocRange();
6129 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6132 if (ID ==
".end_amdhsa_kernel")
6135 if (!Seen.
insert(ID).second)
6136 return TokError(
".amdhsa_ directives cannot be repeated");
6138 SMLoc ValStart = getLoc();
6139 const MCExpr *ExprVal;
6140 if (getParser().parseExpression(ExprVal))
6142 SMLoc ValEnd = getLoc();
6143 SMRange ValRange = SMRange(ValStart, ValEnd);
6147 bool EvaluatableExpr;
6148 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6150 return OutOfRangeError(ValRange);
6154#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6155 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6156 return OutOfRangeError(RANGE); \
6157 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6162#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6164 return Error(IDRange.Start, "directive should have resolvable expression", \
6167 if (ID ==
".amdhsa_group_segment_fixed_size") {
6170 return OutOfRangeError(ValRange);
6172 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6175 return OutOfRangeError(ValRange);
6177 }
else if (ID ==
".amdhsa_kernarg_size") {
6179 return OutOfRangeError(ValRange);
6181 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6183 ExplicitUserSGPRCount = Val;
6184 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6188 "directive is not supported with architected flat scratch",
6191 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6194 ImpliedUserSGPRCount += 4;
6195 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6198 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6201 return OutOfRangeError(ValRange);
6205 ImpliedUserSGPRCount += Val;
6206 PreloadLength = Val;
6208 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6211 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6214 return OutOfRangeError(ValRange);
6218 PreloadOffset = Val;
6219 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6222 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6225 ImpliedUserSGPRCount += 2;
6226 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6229 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6232 ImpliedUserSGPRCount += 2;
6233 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6236 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6239 ImpliedUserSGPRCount += 2;
6240 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6243 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6246 ImpliedUserSGPRCount += 2;
6247 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6250 "directive is not supported with architected flat scratch",
6254 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6257 ImpliedUserSGPRCount += 2;
6258 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6261 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6264 ImpliedUserSGPRCount += 1;
6265 }
else if (ID ==
".amdhsa_wavefront_size32") {
6268 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6269 EnableWavefrontSize32 = Val;
6271 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6273 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6275 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6277 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6280 "directive is not supported with architected flat scratch",
6283 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6285 }
else if (ID ==
".amdhsa_enable_private_segment") {
6289 "directive is not supported without architected flat scratch",
6292 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6294 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6296 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6298 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6300 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6302 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6304 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6306 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6308 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6310 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6312 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6314 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6315 VGPRRange = ValRange;
6316 NextFreeVGPR = ExprVal;
6317 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6318 SGPRRange = ValRange;
6319 NextFreeSGPR = ExprVal;
6320 }
else if (ID ==
".amdhsa_accum_offset") {
6322 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6323 AccumOffset = ExprVal;
6324 }
else if (ID ==
".amdhsa_named_barrier_count") {
6326 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6327 NamedBarCnt = ExprVal;
6328 }
else if (ID ==
".amdhsa_reserve_vcc") {
6330 return OutOfRangeError(ValRange);
6331 ReserveVCC = ExprVal;
6332 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6334 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6337 "directive is not supported with architected flat scratch",
6340 return OutOfRangeError(ValRange);
6341 ReserveFlatScr = ExprVal;
6342 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6344 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6346 return OutOfRangeError(ValRange);
6347 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6348 if (Val != XnackOn) {
6349 return getParser().Error(
6351 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6353 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6355 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6357 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6359 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6361 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6363 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6365 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6367 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6369 }
else if (ID ==
".amdhsa_dx10_clamp") {
6370 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6371 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6374 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6376 }
else if (ID ==
".amdhsa_ieee_mode") {
6377 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6378 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6381 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6383 }
else if (ID ==
".amdhsa_fp16_overflow") {
6385 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6387 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6389 }
else if (ID ==
".amdhsa_tg_split") {
6391 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6394 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6397 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6399 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6401 }
else if (ID ==
".amdhsa_memory_ordered") {
6403 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6405 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6407 }
else if (ID ==
".amdhsa_forward_progress") {
6409 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6411 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6413 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6415 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6416 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6418 SharedVGPRCount = Val;
6420 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6422 }
else if (ID ==
".amdhsa_inst_pref_size") {
6424 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6425 if (
ISA.Major == 11) {
6427 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6431 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6434 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6437 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6439 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6441 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6443 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6446 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6448 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6450 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6452 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6454 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6456 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6458 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6460 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6462 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6464 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6466 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6468 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6471 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6474#undef PARSE_BITS_ENTRY
6477 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6478 return TokError(
".amdhsa_next_free_vgpr directive is required");
6480 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6481 return TokError(
".amdhsa_next_free_sgpr directive is required");
6483 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6485 return TokError(
"too many user SGPRs enabled, found " +
6486 Twine(UserSGPRCount) +
", but only " +
6492 if (PreloadLength) {
6498 const MCExpr *VGPRBlocks;
6499 const MCExpr *SGPRBlocks;
6500 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6501 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6502 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6503 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6506 int64_t EvaluatedVGPRBlocks;
6507 bool VGPRBlocksEvaluatable =
6508 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6509 if (VGPRBlocksEvaluatable &&
6511 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6512 return OutOfRangeError(VGPRRange);
6516 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6517 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6519 int64_t EvaluatedSGPRBlocks;
6520 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6522 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6523 return OutOfRangeError(SGPRRange);
6526 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6527 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6529 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6530 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6531 "enabled user SGPRs");
6537 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6538 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6543 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6544 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6549 return TokError(
"Kernarg size should be resolvable");
6551 if (PreloadLength && kernarg_size &&
6552 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6553 return TokError(
"Kernarg preload length + offset is larger than the "
6554 "kernarg segment size");
6557 if (!Seen.
contains(
".amdhsa_accum_offset"))
6558 return TokError(
".amdhsa_accum_offset directive is required");
6559 int64_t EvaluatedAccum;
6560 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6561 uint64_t UEvaluatedAccum = EvaluatedAccum;
6562 if (AccumEvaluatable &&
6563 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6564 return TokError(
"accum_offset should be in range [4..256] in "
6567 int64_t EvaluatedNumVGPR;
6568 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6572 return TokError(
"accum_offset exceeds total VGPR allocation");
6578 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6579 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6585 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6586 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6589 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6591 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6592 return TokError(
"shared_vgpr_count directive not valid on "
6593 "wavefront size 32");
6596 if (VGPRBlocksEvaluatable &&
6597 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6599 return TokError(
"shared_vgpr_count*2 + "
6600 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6605 emitTargetDirective();
6606 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6607 NextFreeVGPR, NextFreeSGPR,
6608 ReserveVCC, ReserveFlatScr);
6612bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6614 if (ParseAsAbsoluteExpression(
Version))
6617 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6618 emitTargetDirective();
6622bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6623 AMDGPUMCKernelCodeT &
C) {
6626 if (ID ==
"max_scratch_backing_memory_byte_size") {
6627 Parser.eatToEndOfStatement();
6631 SmallString<40> ErrStr;
6632 raw_svector_ostream Err(ErrStr);
6633 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6634 return TokError(Err.
str());
6638 if (ID ==
"enable_wavefront_size32") {
6641 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6643 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6646 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6650 if (ID ==
"wavefront_size") {
6651 if (
C.wavefront_size == 5) {
6653 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6655 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6656 }
else if (
C.wavefront_size == 6) {
6658 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6665bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6666 AMDGPUMCKernelCodeT KernelCode;
6676 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6679 if (ID ==
".end_amd_kernel_code_t")
6682 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6687 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6692bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6693 StringRef KernelName;
6694 if (!parseId(KernelName,
"expected symbol name"))
6697 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6704bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6705 if (!getSTI().getTargetTriple().isAMDGCN()) {
6706 return Error(getLoc(),
6707 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6711 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6713 std::optional<AMDGPU::TargetID> MaybeParsed =
6716 return Error(getParser().getTok().getLoc(),
6717 "malformed target id '" + TargetIDDirective +
"'");
6720 const Triple &
TT = getSTI().getTargetTriple();
6726 return Error(getParser().getTok().getLoc(),
6727 "target id '" + TargetIDDirective +
6728 "' specifies a processor that is not valid for subarch '" +
6729 TT.getArchName() +
"'");
6732 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6733 getTargetStreamer().getTargetID();
6736 const Triple &STITriple = getSTI().getTargetTriple();
6737 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6738 return Error(getParser().getTok().getLoc(),
6739 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6740 " is incompatible with " +
6741 Twine(CurrentTargetID->toString()));
6745 StringRef DirectiveProcessor =
6748 if (DirectiveISA != ISA) {
6749 return Error(getParser().getTok().getLoc(),
6750 ".amd_amdgpu_isa directive processor " +
6751 Twine(DirectiveProcessor) +
6752 " does not match the specified processor " +
6753 Twine(getSTI().
getCPU()));
6756 getTargetStreamer().EmitISAVersion();
6762bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6765 std::string HSAMetadataString;
6770 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6771 return Error(getLoc(),
"invalid HSA metadata");
6778bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6779 const char *AssemblerDirectiveEnd,
6780 std::string &CollectString) {
6782 raw_string_ostream CollectStream(CollectString);
6784 getLexer().setSkipSpace(
false);
6786 bool FoundEnd =
false;
6789 CollectStream << getTokenStr();
6793 if (trySkipId(AssemblerDirectiveEnd)) {
6798 CollectStream << Parser.parseStringToEndOfStatement()
6799 <<
getContext().getAsmInfo().getSeparatorString();
6801 Parser.eatToEndOfStatement();
6804 getLexer().setSkipSpace(
true);
6807 return TokError(Twine(
"expected directive ") +
6808 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6815bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6821 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6822 if (!PALMetadata->setFromString(
String))
6823 return Error(getLoc(),
"invalid PAL metadata");
6828bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6831 Twine(
" directive is "
6832 "not available on non-amdpal OSes"))
6836 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6837 PALMetadata->setLegacy();
6840 if (ParseAsAbsoluteExpression(
Key)) {
6841 return TokError(Twine(
"invalid value in ") +
6845 return TokError(Twine(
"expected an even number of values in ") +
6848 if (ParseAsAbsoluteExpression(
Value)) {
6849 return TokError(Twine(
"invalid value in ") +
6852 PALMetadata->setRegister(
Key,
Value);
6861bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6862 if (getParser().checkForValidSection())
6866 SMLoc NameLoc = getLoc();
6867 if (getParser().parseIdentifier(Name))
6868 return TokError(
"expected identifier in directive");
6871 if (getParser().parseComma())
6877 SMLoc SizeLoc = getLoc();
6878 if (getParser().parseAbsoluteExpression(
Size))
6881 return Error(SizeLoc,
"size must be non-negative");
6882 if (
Size > LocalMemorySize)
6883 return Error(SizeLoc,
"size is too large");
6887 SMLoc AlignLoc = getLoc();
6888 if (getParser().parseAbsoluteExpression(Alignment))
6891 return Error(AlignLoc,
"alignment must be a power of two");
6896 if (Alignment >= 1u << 31)
6897 return Error(AlignLoc,
"alignment is too large");
6903 Symbol->redefineIfPossible();
6904 if (!
Symbol->isUndefined())
6905 return Error(NameLoc,
"invalid symbol redefinition");
6907 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6911bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6912 if (getParser().checkForValidSection())
6916 if (getParser().parseIdentifier(FuncName))
6917 return TokError(
"expected symbol name after .amdgpu_info");
6920 AMDGPU::InfoSectionData ParsedInfoData;
6921 AMDGPU::FuncInfo FI;
6923 bool HasScalarAttrs =
false;
6930 SMLoc IDLoc = getLoc();
6931 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6934 if (ID ==
".end_amdgpu_info")
6942 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6944 if (Dir ==
"flags") {
6946 if (getParser().parseAbsoluteExpression(Val))
6949 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6951 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6953 HasScalarAttrs =
true;
6954 }
else if (Dir ==
"num_sgpr") {
6956 if (getParser().parseAbsoluteExpression(Val))
6958 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6959 HasScalarAttrs =
true;
6960 }
else if (Dir ==
"num_vgpr") {
6962 if (getParser().parseAbsoluteExpression(Val))
6965 HasScalarAttrs =
true;
6966 }
else if (Dir ==
"num_agpr") {
6968 if (getParser().parseAbsoluteExpression(Val))
6971 HasScalarAttrs =
true;
6972 }
else if (Dir ==
"private_segment_size") {
6974 if (getParser().parseAbsoluteExpression(Val))
6977 HasScalarAttrs =
true;
6978 }
else if (Dir ==
"use") {
6980 if (getParser().parseIdentifier(ResName))
6981 return TokError(
"expected resource symbol for .amdgpu_use");
6982 ParsedInfoData.
Uses.push_back(
6983 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6984 }
else if (Dir ==
"call") {
6986 if (getParser().parseIdentifier(DstName))
6987 return TokError(
"expected callee symbol for .amdgpu_call");
6988 ParsedInfoData.
Calls.push_back(
6989 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6990 }
else if (Dir ==
"indirect_call") {
6992 if (getParser().parseEscapedString(TypeId))
6993 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6994 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6995 }
else if (Dir ==
"typeid") {
6997 if (getParser().parseEscapedString(TypeId))
6998 return TokError(
"expected type ID string for .amdgpu_typeid");
6999 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
7001 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
7006 ParsedInfoData.
Funcs.push_back(std::move(FI));
7008 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
7009 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
7010 Data.Funcs.push_back(std::move(Func));
7011 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
7012 Data.Uses.push_back(Use);
7013 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
7015 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
7018 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
7019 Data.TypeIds.push_back(std::move(TypeId));
7024void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7031void AMDGPUAsmParser::checkKernelPrologues() {
7032 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7033 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7034 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7035 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7036 if (!AMDHSAKernelSymbols.
contains(Sym))
7038 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7039 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7043 "' does not begin with the required prologue "
7044 "sequence: s_mov_b64 followed by v_nop and "
7045 "global_prefetch_b8");
7049 OpcodeStream.
clear();
7050 OpcodeStreamSymbols.clear();
7051 AMDHSAKernelSymbols.
clear();
7054void AMDGPUAsmParser::onEndOfFile() {
7055 emitTargetDirective();
7056 checkKernelPrologues();
7058 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7061bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7062 StringRef IDVal = DirectiveID.
getString();
7065 if (IDVal ==
".amdhsa_kernel")
7066 return ParseDirectiveAMDHSAKernel();
7068 if (IDVal ==
".amdhsa_code_object_version")
7069 return ParseDirectiveAMDHSACodeObjectVersion();
7073 return ParseDirectiveHSAMetadata();
7075 if (IDVal ==
".amd_kernel_code_t")
7076 return ParseDirectiveAMDKernelCodeT();
7078 if (IDVal ==
".amdgpu_hsa_kernel")
7079 return ParseDirectiveAMDGPUHsaKernel();
7081 if (IDVal ==
".amd_amdgpu_isa")
7082 return ParseDirectiveISAVersion();
7086 Twine(
" directive is "
7087 "not available on non-amdhsa OSes"))
7092 if (IDVal ==
".amdgcn_target")
7093 return ParseDirectiveAMDGCNTarget();
7095 if (IDVal ==
".amdgpu_lds")
7096 return ParseDirectiveAMDGPULDS();
7098 if (IDVal ==
".amdgpu_info")
7099 return ParseDirectiveAMDGPUInfo();
7102 return ParseDirectivePALMetadataBegin();
7105 return ParseDirectivePALMetadata();
7110bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7117 return hasSGPR104_SGPR105();
7120 case SRC_SHARED_BASE_LO:
7121 case SRC_SHARED_BASE:
7122 case SRC_SHARED_LIMIT_LO:
7123 case SRC_SHARED_LIMIT:
7125 case SRC_PRIVATE_BASE_LO:
7126 case SRC_PRIVATE_BASE:
7127 case SRC_PRIVATE_LIMIT_LO:
7128 case SRC_PRIVATE_LIMIT:
7130 case SRC_FLAT_SCRATCH_BASE_LO:
7131 case SRC_FLAT_SCRATCH_BASE_HI:
7132 return hasGloballyAddressableScratch();
7133 case SRC_POPS_EXITING_WAVE_ID:
7146 getTargetStreamer().getTargetID()->isXnackSupported();
7176 return hasSGPR102_SGPR103();
7184 ParseStatus Res = parseVOPD(
Operands);
7189 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7201 SMLoc LBraceLoc = getLoc();
7206 auto Loc = getLoc();
7209 Error(Loc,
"expected a register");
7213 RBraceLoc = getLoc();
7218 "expected a comma or a closing square bracket"))
7222 if (
Operands.size() - Prefix > 1) {
7224 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7225 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7234StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7236 setForcedEncodingSize(0);
7237 setForcedDPP(
false);
7238 setForcedSDWA(
false);
7240 if (
Name.consume_back(
"_e64_dpp")) {
7242 setForcedEncodingSize(64);
7245 if (
Name.consume_back(
"_e64")) {
7246 setForcedEncodingSize(64);
7249 if (
Name.consume_back(
"_e32")) {
7250 setForcedEncodingSize(32);
7253 if (
Name.consume_back(
"_dpp")) {
7257 if (
Name.consume_back(
"_sdwa")) {
7258 setForcedSDWA(
true);
7266 unsigned VariantID);
7272 Name = parseMnemonicSuffix(Name);
7278 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7280 bool IsMIMG = Name.starts_with(
"image_");
7283 OperandMode
Mode = OperandMode_Default;
7285 Mode = OperandMode_NSA;
7289 checkUnsupportedInstruction(Name, NameLoc);
7290 if (!Parser.hasPendingError()) {
7293 :
"not a valid operand.";
7313ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7316 if (!trySkipId(Name))
7319 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7323ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7332ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7334 std::function<
bool(int64_t &)> ConvertResult) {
7338 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7342 if (ConvertResult && !ConvertResult(
Value)) {
7343 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7346 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7350ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7352 bool (*ConvertResult)(int64_t &)) {
7361 const unsigned MaxSize = 4;
7365 for (
int I = 0;; ++
I) {
7367 SMLoc Loc = getLoc();
7371 if (
Op != 0 &&
Op != 1)
7372 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7379 if (
I + 1 == MaxSize)
7380 return Error(getLoc(),
"expected a closing square bracket");
7386 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7390ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7392 AMDGPUOperand::ImmTy ImmTy,
7393 bool IgnoreNegative) {
7397 if (trySkipId(Name)) {
7399 }
else if (trySkipId(
"no", Name)) {
7408 return Error(S,
"r128 modifier is not supported on this GPU");
7409 if (Name ==
"a16" && !
hasA16())
7410 return Error(S,
"a16 modifier is not supported on this GPU");
7412 if (Bit == 0 && Name ==
"gds") {
7415 return Error(S,
"nogds is not allowed");
7418 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7419 ImmTy = AMDGPUOperand::ImmTyR128A16;
7421 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7425unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7426 bool &Disabling)
const {
7427 Disabling =
Id.consume_front(
"no");
7430 return StringSwitch<unsigned>(Id)
7437 return StringSwitch<unsigned>(Id)
7447 SMLoc StringLoc = getLoc();
7449 int64_t CPolVal = 0;
7470 ResScope = parseScope(
Operands, Scope);
7483 if (trySkipId(
"nv")) {
7487 }
else if (trySkipId(
"no",
"nv")) {
7494 if (trySkipId(
"scale_offset")) {
7498 }
else if (trySkipId(
"no",
"scale_offset")) {
7522 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7523 AMDGPUOperand::ImmTyCPol));
7528 SMLoc OpLoc = getLoc();
7529 unsigned Enabled = 0, Seen = 0;
7533 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7540 return Error(S,
"dlc modifier is not supported on this GPU");
7543 return Error(S,
"scc modifier is not supported on this GPU");
7546 return Error(S,
"duplicate cache policy modifier");
7558 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7567 ParseStatus Res = parseStringOrIntWithPrefix(
7568 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7582 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7586 if (
Value ==
"TH_DEFAULT")
7588 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7589 Value ==
"TH_LOAD_NT_WB") {
7590 return Error(StringLoc,
"invalid th value");
7591 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7593 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7595 }
else if (
Value.consume_front(
"TH_STORE_")) {
7598 return Error(StringLoc,
"invalid th value");
7601 if (
Value ==
"BYPASS")
7606 TH |= StringSwitch<int64_t>(
Value)
7616 .Default(0xffffffff);
7618 TH |= StringSwitch<int64_t>(
Value)
7629 .Default(0xffffffff);
7632 if (TH == 0xffffffff)
7633 return Error(StringLoc,
"invalid th value");
7642 ParseStatus Res = parseStringOrIntWithPrefix(
7643 Operands,
"cfs", {
"CFS_256B",
"CFS_128B",
"CFS_64B",
"CFS_32B"},
CFS);
7653 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7654 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7655 std::optional<unsigned> InsertAt = std::nullopt) {
7656 auto i = OptionalIdx.find(ImmT);
7657 if (i != OptionalIdx.end()) {
7658 unsigned Idx = i->second;
7659 const AMDGPUOperand &
Op =
7660 static_cast<const AMDGPUOperand &
>(*
Operands[
Idx]);
7664 Op.addImmOperands(Inst, 1);
7666 if (InsertAt.has_value())
7673ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7679 StringLoc = getLoc();
7684ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7690 SMLoc StringLoc = getLoc();
7694 Value = getTokenStr();
7698 if (
Value == Ids[IntVal])
7703 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7704 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7709ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7711 AMDGPUOperand::ImmTy
Type) {
7715 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7717 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7726bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7729 SMLoc Loc = getLoc();
7731 auto Res = parseIntWithPrefix(Pref, Val);
7737 if (Val < 0 || Val > MaxVal) {
7738 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7747 AMDGPUOperand::ImmTy ImmTy) {
7748 const char *Pref =
"index_key";
7750 SMLoc Loc = getLoc();
7751 auto Res = parseIntWithPrefix(Pref, ImmVal);
7755 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7756 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7757 (ImmVal < 0 || ImmVal > 1))
7758 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7760 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7761 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7763 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7768 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7772 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7776 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7781 AMDGPUOperand::ImmTy
Type) {
7787 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7788 AMDGPUOperand::ImmTyMatrixAFMT);
7792 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7793 AMDGPUOperand::ImmTyMatrixBFMT);
7798 AMDGPUOperand::ImmTy
Type) {
7804 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7805 AMDGPUOperand::ImmTyMatrixAScale);
7809 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7810 AMDGPUOperand::ImmTyMatrixBScale);
7815 AMDGPUOperand::ImmTy
Type) {
7821 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7822 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7826 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7827 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7832ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7833 using namespace llvm::AMDGPU::MTBUFFormat;
7839 for (
int I = 0;
I < 2; ++
I) {
7840 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7843 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7848 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7854 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7857 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7858 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7864ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7865 using namespace llvm::AMDGPU::MTBUFFormat;
7869 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7872 if (Fmt == UFMT_UNDEF)
7879bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7880 StringRef FormatStr, SMLoc Loc) {
7881 using namespace llvm::AMDGPU::MTBUFFormat;
7885 if (
Format != DFMT_UNDEF) {
7891 if (
Format != NFMT_UNDEF) {
7896 Error(Loc,
"unsupported format");
7900ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7903 using namespace llvm::AMDGPU::MTBUFFormat;
7907 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7912 SMLoc Loc = getLoc();
7913 if (!parseId(Str,
"expected a format string") ||
7914 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7916 if (Dfmt == DFMT_UNDEF)
7917 return Error(Loc,
"duplicate numeric format");
7918 if (Nfmt == NFMT_UNDEF)
7919 return Error(Loc,
"duplicate data format");
7922 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7923 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7927 if (Ufmt == UFMT_UNDEF)
7928 return Error(FormatLoc,
"unsupported format");
7937ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7940 using namespace llvm::AMDGPU::MTBUFFormat;
7943 if (Id == UFMT_UNDEF)
7947 return Error(Loc,
"unified format is not supported on this GPU");
7953ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7954 using namespace llvm::AMDGPU::MTBUFFormat;
7955 SMLoc Loc = getLoc();
7960 return Error(Loc,
"out of range format");
7965ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7966 using namespace llvm::AMDGPU::MTBUFFormat;
7972 StringRef FormatStr;
7973 SMLoc Loc = getLoc();
7974 if (!parseId(FormatStr,
"expected a format string"))
7977 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7979 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7989 return parseNumericFormat(
Format);
7993 using namespace llvm::AMDGPU::MTBUFFormat;
7997 SMLoc Loc = getLoc();
8007 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
8026 Res = parseSymbolicOrNumericFormat(
Format);
8031 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
8032 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
8039 return Error(getLoc(),
"duplicate format");
8045 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
8047 Res = parseIntWithPrefix(
"inst_offset",
Operands,
8048 AMDGPUOperand::ImmTyInstOffset);
8055 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8057 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8063 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8066 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8076 OptionalImmIndexMap OptionalIdx;
8078 unsigned OperandIdx[4];
8079 unsigned EnMask = 0;
8082 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8083 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8088 OperandIdx[SrcIdx] = Inst.
size();
8089 Op.addRegOperands(Inst, 1);
8096 OperandIdx[SrcIdx] = Inst.
size();
8102 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8103 Op.addImmOperands(Inst, 1);
8107 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8111 OptionalIdx[
Op.getImmTy()] = i;
8117 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8124 for (
auto i = 0; i < SrcIdx; ++i) {
8126 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8132 AMDGPUOperand::ImmTyExpCompr);
8142 int64_t CntVal,
bool Saturate,
8148 IntVal =
encode(ISA, IntVal, CntVal);
8149 if (CntVal !=
decode(ISA, IntVal)) {
8151 IntVal =
encode(ISA, IntVal, -1);
8159bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8161 SMLoc CntLoc = getLoc();
8162 StringRef CntName = getTokenStr();
8169 SMLoc ValLoc = getLoc();
8176 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8178 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8180 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8183 Error(CntLoc,
"invalid counter name " + CntName);
8188 Error(ValLoc,
"too large value for " + CntName);
8197 Error(getLoc(),
"expected a counter name");
8211 if (!parseCnt(Waitcnt))
8219 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8223bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8224 SMLoc FieldLoc = getLoc();
8225 StringRef FieldName = getTokenStr();
8230 SMLoc ValueLoc = getLoc();
8237 if (FieldName ==
"instid0") {
8239 }
else if (FieldName ==
"instskip") {
8241 }
else if (FieldName ==
"instid1") {
8244 Error(FieldLoc,
"invalid field name " + FieldName);
8263 .Case(
"VALU_DEP_1", 1)
8264 .Case(
"VALU_DEP_2", 2)
8265 .Case(
"VALU_DEP_3", 3)
8266 .Case(
"VALU_DEP_4", 4)
8267 .Case(
"TRANS32_DEP_1", 5)
8268 .Case(
"TRANS32_DEP_2", 6)
8269 .Case(
"TRANS32_DEP_3", 7)
8270 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8271 .Case(
"SALU_CYCLE_1", 9)
8272 .Case(
"SALU_CYCLE_2", 10)
8273 .Case(
"SALU_CYCLE_3", 11)
8281 Delay |=
Value << Shift;
8291 if (!parseDelay(Delay))
8299 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8303bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8305bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8311void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8312 StringRef DepCtrName) {
8315 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8318 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8321 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8324 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8331bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8333 using namespace llvm::AMDGPU::DepCtr;
8335 SMLoc DepCtrLoc = getLoc();
8336 StringRef DepCtrName = getTokenStr();
8346 unsigned PrevOprMask = UsedOprMask;
8347 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8350 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8359 Error(getLoc(),
"expected a counter name");
8364 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8365 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8370 using namespace llvm::AMDGPU::DepCtr;
8373 SMLoc Loc = getLoc();
8376 unsigned UsedOprMask = 0;
8378 if (!parseDepCtr(DepCtr, UsedOprMask))
8386 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8390bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8396ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8398 OperandInfoTy &Width) {
8399 using namespace llvm::AMDGPU::Hwreg;
8405 HwReg.Loc = getLoc();
8408 HwReg.IsSymbolic =
true;
8410 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8418 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8428 Width.Loc = getLoc();
8437 using namespace llvm::AMDGPU::Hwreg;
8440 SMLoc Loc = getLoc();
8442 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8444 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8445 HwregOffset::Default);
8446 struct : StructuredOpField {
8447 using StructuredOpField::StructuredOpField;
8448 bool validate(AMDGPUAsmParser &Parser)
const override {
8450 return Error(Parser,
"only values from 1 to 32 are legal");
8453 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8454 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8457 Res = parseHwregFunc(HwReg,
Offset, Width);
8460 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8462 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8466 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8473 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8475 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8479bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8485bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8486 OperandInfoTy &Stream) {
8487 using namespace llvm::AMDGPU::SendMsg;
8492 Msg.IsSymbolic =
true;
8499 Op.IsDefined =
true;
8505 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8510 Stream.IsDefined =
true;
8511 Stream.Loc = getLoc();
8520bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8521 const OperandInfoTy &
Op,
8522 const OperandInfoTy &Stream) {
8523 using namespace llvm::AMDGPU::SendMsg;
8532 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8537 Error(
Msg.Loc,
"invalid message id");
8543 Error(
Op.Loc,
"message does not support operations");
8545 Error(
Msg.Loc,
"missing message operation");
8551 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8553 Error(
Op.Loc,
"invalid operation id");
8558 Error(Stream.Loc,
"message operation does not support streams");
8562 Error(Stream.Loc,
"invalid message stream id");
8569 using namespace llvm::AMDGPU::SendMsg;
8572 SMLoc Loc = getLoc();
8576 OperandInfoTy
Op(OP_NONE_);
8577 OperandInfoTy Stream(STREAM_ID_NONE_);
8578 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8583 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8585 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8591 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8595bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8598 using namespace llvm::AMDGPU::WaitEvent;
8600 SMLoc Loc = getLoc();
8603 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8605 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8607 StructuredOpField *TargetBitfield =
8608 isGFX11() ? &DontWaitExportReady : &ExportReady;
8610 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8614 if (!validateStructuredOpFields({TargetBitfield}))
8616 ImmVal = TargetBitfield->Val;
8623 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8625 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8626 AMDGPUOperand::ImmTyWaitEvent));
8630bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8643 int Slot = StringSwitch<int>(Str)
8650 return Error(S,
"invalid interpolation slot");
8653 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8664 if (!Str.starts_with(
"attr"))
8665 return Error(S,
"invalid interpolation attribute");
8667 StringRef Chan = Str.take_back(2);
8668 int AttrChan = StringSwitch<int>(Chan)
8675 return Error(S,
"invalid or missing interpolation attribute channel");
8677 Str = Str.drop_back(2).drop_front(4);
8680 if (Str.getAsInteger(10, Attr))
8681 return Error(S,
"invalid or missing interpolation attribute number");
8684 return Error(S,
"out of bounds interpolation attribute number");
8689 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8690 Operands.push_back(AMDGPUOperand::CreateImm(
8691 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8700 using namespace llvm::AMDGPU::Exp;
8710 return Error(S, (Id == ET_INVALID)
8711 ?
"invalid exp target"
8712 :
"exp target is not supported on this GPU");
8715 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8723bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8727bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8732 return getTokenKind() ==
Kind;
8735StringRef AMDGPUAsmParser::getId()
const {
8739bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8747bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8749 StringRef Tok = getTokenStr();
8758bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8760 if (isId(Id) && peekToken().is(Kind)) {
8769 if (isToken(Kind)) {
8777 const StringRef ErrMsg) {
8778 if (!trySkipToken(Kind)) {
8779 Error(getLoc(), ErrMsg);
8785bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8789 if (Parser.parseExpression(Expr))
8792 if (Expr->evaluateAsAbsolute(
Imm))
8795 if (Expected.empty()) {
8796 Error(S,
"expected absolute expression");
8799 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8808 if (Parser.parseExpression(Expr))
8812 if (Expr->evaluateAsAbsolute(IntVal)) {
8813 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8815 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8820bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8822 Val =
getToken().getStringContents();
8826 Error(getLoc(), ErrMsg);
8830bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8832 Val = getTokenStr();
8836 if (!ErrMsg.
empty())
8837 Error(getLoc(), ErrMsg);
8841AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8843AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8846 : getLexer().peekTok(ShouldSkipSpace);
8850 auto TokCount = getLexer().peekTokens(Tokens);
8857 return getLexer().getKind();
8860SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8862StringRef AMDGPUAsmParser::getTokenStr()
const {
8866void AMDGPUAsmParser::lex() { Parser.Lex(); }
8868const AMDGPUOperand &
8870 int MCOpIdx)
const {
8872 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8873 if (TargetOp.getMCOpIdx() == MCOpIdx)
8880 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8884SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8889 int MCOpIdx)
const {
8890 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8893SMLoc AMDGPUAsmParser::getOperandLoc(
8894 std::function<
bool(
const AMDGPUOperand &)>
Test,
8896 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8897 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8899 return Op.getStartLoc();
8904SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8906 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8921 StringRef
Id = getTokenStr();
8922 SMLoc IdLoc = getLoc();
8928 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8929 if (
I == Fields.
end())
8930 return Error(IdLoc,
"unknown field");
8931 if ((*I)->IsDefined)
8932 return Error(IdLoc,
"duplicate field");
8935 (*I)->Loc = getLoc();
8938 (*I)->IsDefined =
true;
8945bool AMDGPUAsmParser::validateStructuredOpFields(
8947 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8948 return F->validate(*
this);
8958 const unsigned XorMask) {
8965bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8966 const unsigned MaxVal,
8967 const Twine &ErrMsg, SMLoc &Loc) {
8983bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8984 const unsigned MinVal,
8985 const unsigned MaxVal,
8986 const StringRef ErrMsg) {
8988 for (
unsigned i = 0; i < OpNum; ++i) {
8989 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8996bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8997 using namespace llvm::AMDGPU::Swizzle;
9000 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
9001 "expected a 2-bit lane id")) {
9011bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
9012 using namespace llvm::AMDGPU::Swizzle;
9018 if (!parseSwizzleOperand(GroupSize, 2, 32,
9019 "group size must be in the interval [2,32]", Loc)) {
9023 Error(Loc,
"group size must be a power of two");
9026 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
9027 "lane id must be in the interval [0,group size - 1]",
9035bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
9036 using namespace llvm::AMDGPU::Swizzle;
9041 if (!parseSwizzleOperand(GroupSize, 2, 32,
9042 "group size must be in the interval [2,32]", Loc)) {
9046 Error(Loc,
"group size must be a power of two");
9054bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9055 using namespace llvm::AMDGPU::Swizzle;
9060 if (!parseSwizzleOperand(GroupSize, 1, 16,
9061 "group size must be in the interval [1,16]", Loc)) {
9065 Error(Loc,
"group size must be a power of two");
9073bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9074 using namespace llvm::AMDGPU::Swizzle;
9081 SMLoc StrLoc = getLoc();
9082 if (!parseString(Ctl)) {
9085 if (Ctl.
size() != BITMASK_WIDTH) {
9086 Error(StrLoc,
"expected a 5-character mask");
9090 unsigned AndMask = 0;
9091 unsigned OrMask = 0;
9092 unsigned XorMask = 0;
9094 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9098 Error(StrLoc,
"invalid mask");
9119bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9120 using namespace llvm::AMDGPU::Swizzle;
9123 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9129 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9130 "FFT swizzle must be in the interval [0," +
9131 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9139bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9140 using namespace llvm::AMDGPU::Swizzle;
9143 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9150 if (!parseSwizzleOperand(
Direction, 0, 1,
9151 "direction must be 0 (left) or 1 (right)", Loc))
9155 if (!parseSwizzleOperand(
9156 RotateSize, 0, ROTATE_MAX_SIZE,
9157 "number of threads to rotate must be in the interval [0," +
9158 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9163 (RotateSize << ROTATE_SIZE_SHIFT);
9167bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9169 SMLoc OffsetLoc = getLoc();
9175 Error(OffsetLoc,
"expected a 16-bit offset");
9181bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9182 using namespace llvm::AMDGPU::Swizzle;
9186 SMLoc ModeLoc = getLoc();
9189 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9190 Ok = parseSwizzleQuadPerm(
Imm);
9191 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9192 Ok = parseSwizzleBitmaskPerm(
Imm);
9193 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9194 Ok = parseSwizzleBroadcast(
Imm);
9195 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9196 Ok = parseSwizzleSwap(
Imm);
9197 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9198 Ok = parseSwizzleReverse(
Imm);
9199 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9200 Ok = parseSwizzleFFT(
Imm);
9201 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9202 Ok = parseSwizzleRotate(
Imm);
9204 Error(ModeLoc,
"expected a swizzle mode");
9207 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9217 if (trySkipId(
"offset")) {
9221 if (trySkipId(
"swizzle")) {
9222 Ok = parseSwizzleMacro(
Imm);
9224 Ok = parseSwizzleOffset(
Imm);
9229 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9236bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9242int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9244 using namespace llvm::AMDGPU::VGPRIndexMode;
9256 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9257 if (trySkipId(IdSymbolic[ModeId])) {
9265 ?
"expected a VGPR index mode or a closing parenthesis"
9266 :
"expected a VGPR index mode");
9271 Error(S,
"duplicate VGPR index mode");
9279 "expected a comma or a closing parenthesis"))
9288 using namespace llvm::AMDGPU::VGPRIndexMode;
9294 Imm = parseGPRIdxMacro();
9298 if (getParser().parseAbsoluteExpression(
Imm))
9301 return Error(S,
"invalid immediate: only 4-bit values are legal");
9305 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9309bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9320 if (isRegister() || isModifier())
9327 assert(Opr.isImm() || Opr.isExpr());
9328 SMLoc Loc = Opr.getStartLoc();
9332 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9333 Error(Loc,
"expected an absolute expression or a label");
9334 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9335 Error(Loc,
"expected a 16-bit signed jump offset");
9355 OptionalImmIndexMap OptionalIdx;
9356 unsigned FirstOperandIdx = 1;
9357 bool IsAtomicReturn =
false;
9363 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9364 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9368 Op.addRegOperands(Inst, 1);
9372 if (IsAtomicReturn && i == FirstOperandIdx)
9373 Op.addRegOperands(Inst, 1);
9378 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9379 Op.addImmOperands(Inst, 1);
9391 OptionalIdx[
Op.getImmTy()] = i;
9395 AMDGPUOperand::ImmTyOffset);
9411bool AMDGPUOperand::isSMRDOffset8()
const {
9415bool AMDGPUOperand::isSMEMOffset()
const {
9417 return isImmLiteral();
9420bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9455bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9456 if (BoundCtrl == 0 || BoundCtrl == 1) {
9464void AMDGPUAsmParser::onBeginOfFile() {
9465 if (!getParser().getStreamer().getTargetStreamer())
9468 if (!getTargetStreamer().getTargetID())
9469 getTargetStreamer().initializeTargetID(getSTI(),
9473void AMDGPUAsmParser::emitTargetDirective() {
9474 if (!getParser().getStreamer().getTargetStreamer() ||
9479 getTargetStreamer().emitDirectiveAMDGCNTargetOnce();
9488bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9492 StringRef TokenId = getTokenStr();
9493 AGVK VK = StringSwitch<AGVK>(TokenId)
9494 .Case(
"max", AGVK::AGVK_Max)
9495 .Case(
"min", AGVK::AGVK_Min)
9496 .Case(
"or", AGVK::AGVK_Or)
9497 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9498 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9499 .Case(
"alignto", AGVK::AGVK_AlignTo)
9500 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9501 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9502 .Default(AGVK::AGVK_None);
9511 if (Exprs.
empty()) {
9513 "empty " + Twine(TokenId) +
" expression");
9516 if (CommaCount + 1 != Exprs.
size()) {
9518 "mismatch of commas in " + Twine(TokenId) +
" expression");
9522 Expected && Exprs.
size() != Expected) {
9523 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9524 Twine(Expected) +
" operands");
9531 if (getParser().parseExpression(Expr, EndLoc))
9535 if (LastTokenWasComma)
9539 "unexpected token in " + Twine(TokenId) +
" expression");
9545 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9549 StringRef
Name = getTokenStr();
9550 if (Name ==
"mul") {
9551 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9555 if (Name ==
"div") {
9556 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9567 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9572 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9573 AMDGPU::OpName::src2};
9581 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9586 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9588 if (
DstOp.isReg() &&
9593 if ((OpSel & (1 << SrcNum)) != 0)
9599void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9606 OptionalImmIndexMap &OptionalIdx) {
9607 cvtVOP3P(Inst,
Operands, OptionalIdx);
9616 &&
Desc.NumOperands > (OpNum + 1)
9618 &&
Desc.operands()[OpNum + 1].RegClass != -1
9620 &&
Desc.getOperandConstraint(OpNum + 1,
9624void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9626 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9627 AMDGPU::OpName::src2};
9628 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9629 AMDGPU::OpName::src1_modifiers,
9630 AMDGPU::OpName::src2_modifiers};
9631 for (
int J = 0; J < 3; ++J) {
9632 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9638 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9641 if ((OpSel & (1 << J)) != 0)
9644 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9651void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9653 OptionalImmIndexMap OptionalIdx;
9658 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9659 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9663 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9665 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9666 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9667 Op.isInterpAttrChan()) {
9669 }
else if (
Op.isImmModifier()) {
9670 OptionalIdx[
Op.getImmTy()] =
I;
9678 AMDGPUOperand::ImmTyHigh);
9682 AMDGPUOperand::ImmTyClamp);
9686 AMDGPUOperand::ImmTyOModSI);
9691 AMDGPUOperand::ImmTyOpSel);
9692 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9695 cvtOpSelHelper(Inst, OpSel);
9700 OptionalImmIndexMap OptionalIdx;
9705 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9706 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9710 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9712 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9713 }
else if (
Op.isImmModifier()) {
9714 OptionalIdx[
Op.getImmTy()] =
I;
9722 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9725 AMDGPUOperand::ImmTyOpSel);
9728 AMDGPUOperand::ImmTyWaitEXP);
9734 cvtOpSelHelper(Inst, OpSel);
9737void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9739 OptionalImmIndexMap OptionalIdx;
9742 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9746 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9747 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9750 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9755 if (NumOperands == CbszOpIdx) {
9760 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9761 }
else if (
Op.isImmModifier()) {
9762 OptionalIdx[
Op.getImmTy()] =
I;
9764 Op.addRegOrImmOperands(Inst, 1);
9769 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9770 if (CbszIdx != OptionalIdx.end()) {
9771 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9775 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9776 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9777 if (BlgpIdx != OptionalIdx.end()) {
9778 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9789 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9790 if (OpselIdx != OptionalIdx.end()) {
9791 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9795 unsigned OpSelHi = 0;
9796 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9797 if (OpselHiIdx != OptionalIdx.end()) {
9798 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9801 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9802 AMDGPU::OpName::src1_modifiers};
9804 for (
unsigned J = 0; J < 2; ++J) {
9805 unsigned ModVal = 0;
9806 if (OpSel & (1 << J))
9808 if (OpSelHi & (1 << J))
9811 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9817 OptionalImmIndexMap &OptionalIdx) {
9822 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9823 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9827 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9829 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9830 }
else if (
Op.isImmModifier()) {
9831 OptionalIdx[
Op.getImmTy()] =
I;
9833 Op.addRegOrImmOperands(Inst, 1);
9839 AMDGPUOperand::ImmTyScaleSel);
9843 AMDGPUOperand::ImmTyClamp);
9849 AMDGPUOperand::ImmTyByteSel);
9854 AMDGPUOperand::ImmTyOModSI);
9861 auto *it = Inst.
begin();
9863 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9872 OptionalImmIndexMap OptionalIdx;
9873 cvtVOP3(Inst,
Operands, OptionalIdx);
9877 OptionalImmIndexMap &OptIdx) {
9882 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9883 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9884 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9885 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9886 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9887 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9888 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9889 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9890 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9891 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9900 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9901 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9904 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9905 if (BitOp3Idx != -1) {
9912 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9913 if (OpSelIdx != -1) {
9917 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9918 if (OpSelHiIdx != -1) {
9919 int DefaultVal =
IsPacked ? -1 : 0;
9925 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9926 if (MatrixAFMTIdx != -1) {
9928 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9932 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9933 if (MatrixBFMTIdx != -1) {
9935 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9938 int MatrixAScaleIdx =
9939 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9940 if (MatrixAScaleIdx != -1) {
9942 AMDGPUOperand::ImmTyMatrixAScale, 0);
9945 int MatrixBScaleIdx =
9946 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9947 if (MatrixBScaleIdx != -1) {
9949 AMDGPUOperand::ImmTyMatrixBScale, 0);
9952 int MatrixAScaleFmtIdx =
9953 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9954 if (MatrixAScaleFmtIdx != -1) {
9956 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9959 int MatrixBScaleFmtIdx =
9960 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9961 if (MatrixBScaleFmtIdx != -1) {
9963 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9968 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9972 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9974 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9978 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9982 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9983 AMDGPU::OpName::src2};
9984 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9985 AMDGPU::OpName::src1_modifiers,
9986 AMDGPU::OpName::src2_modifiers};
9989 unsigned OpSelHi = 0;
9996 if (OpSelHiIdx != -1)
10002 if (NegHiIdx != -1)
10005 for (
int J = 0; J < 3; ++J) {
10006 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
10010 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
10020 uint32_t ModVal = 0;
10022 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
10023 if (SrcOp.
isReg() && getMRI()
10027 if (VGPRSuffixIsHi)
10030 if ((OpSel & (1 << J)) != 0)
10034 if ((OpSelHi & (1 << J)) != 0)
10037 if ((NegLo & (1 << J)) != 0)
10040 if ((NegHi & (1 << J)) != 0)
10048 OptionalImmIndexMap OptIdx;
10054 unsigned i,
unsigned Opc,
10055 AMDGPU::OpName
OpName) {
10056 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10057 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10059 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10065 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10068 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10069 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10071 OptionalImmIndexMap OptIdx;
10072 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10073 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10074 OptIdx[
Op.getImmTy()] = i;
10079 AMDGPUOperand::ImmTyIndexKey8bit);
10083 AMDGPUOperand::ImmTyIndexKey16bit);
10087 AMDGPUOperand::ImmTyIndexKey32bit);
10104 SMLoc S = getLoc();
10107 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10108 SMLoc OpYLoc = getLoc();
10111 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10114 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10123 auto addOp = [&](uint16_t ParsedOprIdx) {
10124 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10126 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10130 Op.addRegOperands(Inst, 1);
10133 if (
Op.isImm() ||
Op.isExpr()) {
10134 Op.addImmOperands(Inst, 1);
10146 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10150 const auto &CInfo = InstInfo[CompIdx];
10151 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10152 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10153 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10154 if (CInfo.hasSrc2Acc())
10155 addOp(CInfo.getIndexOfDstInParsedOperands());
10159 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10160 if (BitOp3Idx != -1) {
10161 OptionalImmIndexMap OptIdx;
10162 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10164 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10174bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10176bool AMDGPUOperand::isDPPCtrl()
const {
10177 using namespace AMDGPU::DPP;
10179 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10182 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10183 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10184 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10185 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10186 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10187 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10188 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10189 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10190 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10191 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10192 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10193 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10202bool AMDGPUOperand::isBLGP()
const {
10206bool AMDGPUOperand::isS16Imm()
const {
10210bool AMDGPUOperand::isU16Imm()
const {
10218bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10223 SMLoc Loc =
getToken().getEndLoc();
10224 Token = std::string(getTokenStr());
10226 if (getLoc() != Loc)
10231 if (!parseId(Suffix))
10235 StringRef DimId = Token;
10250 SMLoc S = getLoc();
10256 SMLoc Loc = getLoc();
10257 if (!parseDimId(Encoding))
10258 return Error(Loc,
"invalid dim value");
10261 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10270 SMLoc S = getLoc();
10279 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10282 for (
size_t i = 0; i < 8; ++i) {
10286 SMLoc Loc = getLoc();
10287 if (getParser().parseAbsoluteExpression(Sels[i]))
10289 if (0 > Sels[i] || 7 < Sels[i])
10290 return Error(Loc,
"expected a 3-bit value");
10293 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10297 for (
size_t i = 0; i < 8; ++i)
10298 DPP8 |= (Sels[i] << (i * 3));
10301 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10305bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10307 if (Ctrl ==
"row_newbcast")
10310 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10313 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10314 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10317 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10318 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10322int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10325 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10329 for (
int i = 0; i < 4; ++i) {
10334 SMLoc Loc = getLoc();
10335 if (getParser().parseAbsoluteExpression(Temp))
10337 if (Temp < 0 || Temp > 3) {
10338 Error(Loc,
"expected a 2-bit value");
10342 Val += (Temp << i * 2);
10345 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10351int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10352 using namespace AMDGPU::DPP;
10357 SMLoc Loc = getLoc();
10359 if (getParser().parseAbsoluteExpression(Val))
10362 struct DppCtrlCheck {
10368 DppCtrlCheck
Check =
10369 StringSwitch<DppCtrlCheck>(Ctrl)
10370 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10371 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10372 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10373 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10374 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10375 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10376 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10377 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10378 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10379 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10383 if (
Check.Ctrl == -1) {
10384 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10392 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10400 using namespace AMDGPU::DPP;
10403 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10406 SMLoc S = getLoc();
10412 if (Ctrl ==
"row_mirror") {
10413 Val = DppCtrl::ROW_MIRROR;
10414 }
else if (Ctrl ==
"row_half_mirror") {
10415 Val = DppCtrl::ROW_HALF_MIRROR;
10418 if (Ctrl ==
"quad_perm") {
10419 Val = parseDPPCtrlPerm();
10421 Val = parseDPPCtrlSel(Ctrl);
10430 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10436 OptionalImmIndexMap OptionalIdx;
10443 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10445 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10446 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10450 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10451 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10455 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10456 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10457 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10458 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10459 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10460 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10461 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10462 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10463 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10469 if (OldIdx == NumOperands) {
10471 constexpr int DST_IDX = 0;
10473 }
else if (Src2ModIdx == NumOperands) {
10483 if (IsVOP3CvtSrDpp) {
10492 if (TiedTo != -1) {
10497 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10499 if (IsDPP8 &&
Op.isDppFI()) {
10502 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10503 }
else if (
Op.isReg()) {
10504 Op.addRegOperands(Inst, 1);
10505 }
else if (
Op.isImm() &&
10507 Op.addImmOperands(Inst, 1);
10508 }
else if (
Op.isImm()) {
10509 OptionalIdx[
Op.getImmTy()] =
I;
10517 AMDGPUOperand::ImmTyClamp);
10523 AMDGPUOperand::ImmTyByteSel);
10528 AMDGPUOperand::ImmTyOModSI);
10531 cvtVOP3P(Inst,
Operands, OptionalIdx);
10533 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10536 AMDGPUOperand::ImmTyOpSel);
10541 AMDGPUOperand::ImmTyDPP8);
10542 using namespace llvm::AMDGPU::DPP;
10546 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10548 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10550 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10552 AMDGPUOperand::ImmTyDppBoundCtrl);
10556 AMDGPUOperand::ImmTyDppFI);
10562 OptionalImmIndexMap OptionalIdx;
10566 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10567 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10574 if (TiedTo != -1) {
10579 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10581 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10589 Op.addImmOperands(Inst, 1);
10591 Op.addRegWithFPInputModsOperands(Inst, 2);
10592 }
else if (
Op.isDppFI()) {
10594 }
else if (
Op.isReg()) {
10595 Op.addRegOperands(Inst, 1);
10601 Op.addRegWithFPInputModsOperands(Inst, 2);
10602 }
else if (
Op.isReg()) {
10603 Op.addRegOperands(Inst, 1);
10604 }
else if (
Op.isDPPCtrl()) {
10605 Op.addImmOperands(Inst, 1);
10606 }
else if (
Op.isImm()) {
10608 OptionalIdx[
Op.getImmTy()] =
I;
10616 using namespace llvm::AMDGPU::DPP;
10620 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10622 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10624 AMDGPUOperand::ImmTyDppBoundCtrl);
10627 AMDGPUOperand::ImmTyDppFI);
10638 AMDGPUOperand::ImmTy
Type) {
10639 return parseStringOrIntWithPrefix(
10641 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10646 return parseStringOrIntWithPrefix(
10647 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10648 AMDGPUOperand::ImmTySDWADstUnused);
10652 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10656 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10659void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10661 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10664void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10666 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10674 SDWAInstType BasicInstType,
bool SkipDstVcc,
10676 using namespace llvm::AMDGPU::SDWA;
10678 OptionalImmIndexMap OptionalIdx;
10679 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10680 bool SkippedVcc =
false;
10684 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10685 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10689 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10690 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10691 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10697 if (BasicInstType == SDWAInstType::VOP2 &&
10703 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10709 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10710 }
else if (
Op.isImm()) {
10712 OptionalIdx[
Op.getImmTy()] =
I;
10716 SkippedVcc =
false;
10720 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10721 Opc != AMDGPU::V_NOP_sdwa_vi) {
10723 switch (BasicInstType) {
10724 case SDWAInstType::VOP1:
10727 AMDGPUOperand::ImmTyClamp, 0);
10731 AMDGPUOperand::ImmTyOModSI, 0);
10735 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10739 AMDGPUOperand::ImmTySDWADstUnused,
10740 DstUnused::UNUSED_PRESERVE);
10743 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10746 case SDWAInstType::VOP2:
10748 AMDGPUOperand::ImmTyClamp, 0);
10752 AMDGPUOperand::ImmTyOModSI, 0);
10755 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10757 AMDGPUOperand::ImmTySDWADstUnused,
10758 DstUnused::UNUSED_PRESERVE);
10760 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10762 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10765 case SDWAInstType::VOPC:
10768 AMDGPUOperand::ImmTyClamp, 0);
10770 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10772 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10779 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10780 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10781 auto *it = Inst.
begin();
10783 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10796#define GET_MATCHER_IMPLEMENTATION
10797#define GET_MNEMONIC_SPELL_CHECKER
10798#define GET_MNEMONIC_CHECKER
10799#include "AMDGPUGenAsmMatcher.inc"
10805 return parseTokenOp(
"addr64",
Operands);
10807 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10809 return parseTokenOp(
"idxen",
Operands);
10811 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10814 return parseTokenOp(
"offen",
Operands);
10816 return parseTokenOp(
"off",
Operands);
10817 case MCK_row_95_en:
10818 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10820 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10822 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10824 return tryCustomParseOperand(
Operands, MCK);
10829unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10835 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10838 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10840 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10842 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10844 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10846 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10848 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10850 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10851 case MCK_row_95_en:
10852 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10860 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10862 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10863 case MCK_SOPPBrTarget:
10864 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10865 case MCK_VReg32OrOff:
10866 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10867 case MCK_InterpSlot:
10868 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10869 case MCK_InterpAttr:
10870 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10871 case MCK_InterpAttrChan:
10872 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10874 case MCK_SReg_64_XEXEC:
10884 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10886 return Match_InvalidOperand;
10895 SMLoc S = getLoc();
10904 return Error(S,
"expected a 16-bit value");
10907 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10911bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10917bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
const int16_t * getRegClassByHwModeTable(unsigned ModeId) const
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
const MCRegisterClass & getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_ABI unsigned getTotalNumVGPRs(GPUKind AK, bool IsWave32)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo)
Is this a scalar (i.e. not packed) bf16 source operand?
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
EnumSet< Modifier > Modifiers
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0