20#include "llvm/IR/IntrinsicsSPIRV.h"
25#define DEBUG_TYPE "spirv-builtins"
29#define GET_BuiltinGroup_DECL
30#include "SPIRVGenTables.inc"
34 InstructionSet::InstructionSet
Set;
40#define GET_DemangledBuiltins_DECL
41#define GET_DemangledBuiltins_IMPL
63 InstructionSet::InstructionSet
Set;
67#define GET_NativeBuiltins_DECL
68#define GET_NativeBuiltins_IMPL
86#define GET_GroupBuiltins_DECL
87#define GET_GroupBuiltins_IMPL
97#define GET_IntelSubgroupsBuiltins_DECL
98#define GET_IntelSubgroupsBuiltins_IMPL
105#define GET_AtomicFloatingBuiltins_DECL
106#define GET_AtomicFloatingBuiltins_IMPL
113#define GET_GroupUniformBuiltins_DECL
114#define GET_GroupUniformBuiltins_IMPL
118 InstructionSet::InstructionSet
Set;
123#define GET_GetBuiltins_DECL
124#define GET_GetBuiltins_IMPL
128 InstructionSet::InstructionSet
Set;
132#define GET_ImageQueryBuiltins_DECL
133#define GET_ImageQueryBuiltins_IMPL
141#define GET_IntegerDotProductBuiltins_DECL
142#define GET_IntegerDotProductBuiltins_IMPL
146 InstructionSet::InstructionSet
Set;
157 InstructionSet::InstructionSet
Set;
165#define GET_ConvertBuiltins_DECL
166#define GET_ConvertBuiltins_IMPL
168using namespace InstructionSet;
169#define GET_VectorLoadStoreBuiltins_DECL
170#define GET_VectorLoadStoreBuiltins_IMPL
172#define GET_CLMemoryScope_DECL
173#define GET_CLSamplerAddressingMode_DECL
174#define GET_CLMemoryFenceFlags_DECL
175#define GET_ExtendedBuiltins_DECL
176#include "SPIRVGenTables.inc"
188 StringRef PassPrefix =
"(anonymous namespace)::";
189 std::string BuiltinName;
192 BuiltinName = DemangledCall.
substr(PassPrefix.
size());
194 BuiltinName = DemangledCall;
197 BuiltinName = BuiltinName.
substr(0, BuiltinName.find(
'('));
200 if (BuiltinName.rfind(
"__spirv_ocl_", 0) == 0)
201 BuiltinName = BuiltinName.
substr(12);
206 std::size_t Pos1 = BuiltinName.
rfind(
'<');
207 if (Pos1 != std::string::npos && BuiltinName.back() ==
'>') {
208 std::size_t Pos2 = BuiltinName.rfind(
' ', Pos1);
209 if (Pos2 == std::string::npos)
213 BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2);
214 BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(
' ') + 1);
241 static const std::regex SpvWithR(
242 "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageWrite|ImageQuerySizeLod|"
244 "SDotKHR|SUDotKHR|SDotAccSatKHR|UDotAccSatKHR|SUDotAccSatKHR|"
245 "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|"
246 "SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|"
248 "UConvert|SConvert|FConvert|SatConvert)[^_]*)(_R[^_]*_?(\\w+)?.*)?");
250 if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 1) {
251 std::ssub_match SubMatch;
252 if (DecorationId && Match.size() > 3) {
257 BuiltinName = SubMatch.str();
274static std::unique_ptr<const SPIRV::IncomingCall>
276 SPIRV::InstructionSet::InstructionSet Set,
283 DemangledCall.
slice(DemangledCall.
find(
'(') + 1, DemangledCall.
find(
')'));
284 BuiltinArgs.
split(BuiltinArgumentTypes,
',', -1,
false);
289 if ((Builtin = SPIRV::lookupBuiltin(BuiltinName, Set)))
290 return std::make_unique<SPIRV::IncomingCall>(
291 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
296 if (BuiltinArgumentTypes.
size() >= 1) {
297 char FirstArgumentType = BuiltinArgumentTypes[0][0];
302 switch (FirstArgumentType) {
305 if (Set == SPIRV::InstructionSet::OpenCL_std)
307 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
315 if (Set == SPIRV::InstructionSet::OpenCL_std)
317 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
324 if (Set == SPIRV::InstructionSet::OpenCL_std ||
325 Set == SPIRV::InstructionSet::GLSL_std_450)
331 if (!Prefix.empty() &&
332 (Builtin = SPIRV::lookupBuiltin(Prefix + BuiltinName, Set)))
333 return std::make_unique<SPIRV::IncomingCall>(
334 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
341 switch (FirstArgumentType) {
362 if (!Suffix.empty() &&
363 (Builtin = SPIRV::lookupBuiltin(BuiltinName + Suffix, Set)))
364 return std::make_unique<SPIRV::IncomingCall>(
365 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
380 assert(
MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST &&
381 MI->getOperand(1).isReg());
382 Register BitcastReg =
MI->getOperand(1).getReg();
396 assert(
DefMI->getOpcode() == TargetOpcode::G_CONSTANT &&
397 DefMI->getOperand(1).isCImm());
398 return DefMI->getOperand(1).getCImm()->getValue().getZExtValue();
410 Register ValueReg =
MI->getOperand(0).getReg();
416 assert(Ty &&
"Type is expected");
428 if (
MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
429 return MI->getOperand(1).getGlobal()->getType();
431 "Blocks in OpenCL C must be traceable to allocation site");
443static std::tuple<Register, SPIRVType *>
449 if (ResultType->
getOpcode() == SPIRV::OpTypeVector) {
464 return std::make_tuple(ResultRegister, BoolType);
475 if (ReturnType->getOpcode() == SPIRV::OpTypeVector) {
486 return MIRBuilder.
buildSelect(ReturnRegister, SourceRegister, TrueConst,
496 if (!DestinationReg.isValid())
501 MIRBuilder.
buildLoad(DestinationReg, PtrRegister, PtrInfo,
Align());
502 return DestinationReg;
511 const std::optional<SPIRV::LinkageType::LinkageType> &LinkageTy = {
512 SPIRV::LinkageType::Import}) {
520 VariableType, MIRBuilder, SPIRV::StorageClass::Input);
526 SPIRV::StorageClass::Input,
nullptr, isConst, LinkageTy,
533 return LoadedRegister;
546static SPIRV::MemorySemantics::MemorySemantics
549 case std::memory_order_relaxed:
550 return SPIRV::MemorySemantics::None;
551 case std::memory_order_acquire:
552 return SPIRV::MemorySemantics::Acquire;
553 case std::memory_order_release:
554 return SPIRV::MemorySemantics::Release;
555 case std::memory_order_acq_rel:
556 return SPIRV::MemorySemantics::AcquireRelease;
557 case std::memory_order_seq_cst:
558 return SPIRV::MemorySemantics::SequentiallyConsistent;
566 case SPIRV::CLMemoryScope::memory_scope_work_item:
567 return SPIRV::Scope::Invocation;
568 case SPIRV::CLMemoryScope::memory_scope_work_group:
569 return SPIRV::Scope::Workgroup;
570 case SPIRV::CLMemoryScope::memory_scope_device:
571 return SPIRV::Scope::Device;
572 case SPIRV::CLMemoryScope::memory_scope_all_svm_devices:
573 return SPIRV::Scope::CrossDevice;
574 case SPIRV::CLMemoryScope::memory_scope_sub_group:
575 return SPIRV::Scope::Subgroup;
588 SPIRV::Scope::Scope Scope,
592 if (CLScopeRegister.
isValid()) {
597 if (CLScope ==
static_cast<unsigned>(Scope)) {
598 MRI->setRegClass(CLScopeRegister, &SPIRV::iIDRegClass);
599 return CLScopeRegister;
607 if (
MRI->getRegClassOrNull(
Reg))
611 SpvType ? GR->
getRegClass(SpvType) : &SPIRV::iIDRegClass);
615 Register PtrRegister,
unsigned &Semantics,
618 if (SemanticsRegister.
isValid()) {
620 std::memory_order Order =
625 if (
static_cast<unsigned>(Order) == Semantics) {
626 MRI->setRegClass(SemanticsRegister, &SPIRV::iIDRegClass);
627 return SemanticsRegister;
640 unsigned Sz =
Call->Arguments.size() - ImmArgs.size();
641 for (
unsigned i = 0; i < Sz; ++i)
642 MIB.addUse(
Call->Arguments[i]);
651 if (
Call->isSpirvOp())
655 "Need 2 arguments for atomic init translation");
667 if (
Call->isSpirvOp())
675 Call->Arguments.size() > 1
679 if (
Call->Arguments.size() > 2) {
681 MemSemanticsReg =
Call->Arguments[2];
684 SPIRV::MemorySemantics::SequentiallyConsistent |
702 if (
Call->isSpirvOp())
710 SPIRV::MemorySemantics::SequentiallyConsistent |
725 if (
Call->isSpirvOp())
729 bool IsCmpxchg =
Call->Builtin->Name.contains(
"cmpxchg");
736 LLT DesiredLLT =
MRI->getType(Desired);
739 SPIRV::OpTypePointer);
742 assert(IsCmpxchg ? ExpectedType == SPIRV::OpTypeInt
743 : ExpectedType == SPIRV::OpTypePointer);
748 auto StorageClass =
static_cast<SPIRV::StorageClass::StorageClass
>(
756 ? SPIRV::MemorySemantics::None
757 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
760 ? SPIRV::MemorySemantics::None
761 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
762 if (
Call->Arguments.size() >= 4) {
764 "Need 5+ args for explicit atomic cmpxchg");
771 if (
static_cast<unsigned>(MemOrdEq) == MemSemEqual)
772 MemSemEqualReg =
Call->Arguments[3];
773 if (
static_cast<unsigned>(MemOrdNeq) == MemSemEqual)
774 MemSemUnequalReg =
Call->Arguments[4];
778 if (!MemSemUnequalReg.
isValid())
782 auto Scope = IsCmpxchg ? SPIRV::Scope::Workgroup : SPIRV::Scope::Device;
783 if (
Call->Arguments.size() >= 6) {
785 "Extra args for explicit atomic cmpxchg");
786 auto ClScope =
static_cast<SPIRV::CLMemoryScope
>(
789 if (ClScope ==
static_cast<unsigned>(Scope))
790 ScopeReg =
Call->Arguments[5];
800 Register Tmp = !IsCmpxchg ?
MRI->createGenericVirtualRegister(DesiredLLT)
801 :
Call->ReturnRegister;
802 if (!
MRI->getRegClassOrNull(Tmp))
826 if (
Call->isSpirvOp())
835 "Too many args for explicit atomic RMW");
836 ScopeRegister =
buildScopeReg(ScopeRegister, SPIRV::Scope::Workgroup,
837 MIRBuilder, GR,
MRI);
840 unsigned Semantics = SPIRV::MemorySemantics::None;
844 Semantics, MIRBuilder, GR);
848 if (
Call->ReturnType->getOpcode() == SPIRV::OpTypeFloat) {
849 if (Opcode == SPIRV::OpAtomicIAdd) {
850 Opcode = SPIRV::OpAtomicFAddEXT;
851 }
else if (Opcode == SPIRV::OpAtomicISub) {
854 Opcode = SPIRV::OpAtomicFAddEXT;
856 MRI->createGenericVirtualRegister(
MRI->getType(ValueReg));
865 ValueReg = NegValueReg;
884 "Wrong number of atomic floating-type builtin");
904 bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;
906 if (
Call->isSpirvOp())
912 unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;
916 Semantics, MIRBuilder, GR);
918 assert((Opcode != SPIRV::OpAtomicFlagClear ||
919 (Semantics != SPIRV::MemorySemantics::Acquire &&
920 Semantics != SPIRV::MemorySemantics::AcquireRelease)) &&
921 "Invalid memory order argument!");
944 if ((Opcode == SPIRV::OpControlBarrierArriveINTEL ||
945 Opcode == SPIRV::OpControlBarrierWaitINTEL) &&
946 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {
947 std::string DiagMsg = std::string(Builtin->
Name) +
948 ": the builtin requires the following SPIR-V "
949 "extension: SPV_INTEL_split_barrier";
953 if (
Call->isSpirvOp())
958 unsigned MemSemantics = SPIRV::MemorySemantics::None;
960 if (MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE)
961 MemSemantics |= SPIRV::MemorySemantics::WorkgroupMemory;
963 if (MemFlags & SPIRV::CLK_GLOBAL_MEM_FENCE)
964 MemSemantics |= SPIRV::MemorySemantics::CrossWorkgroupMemory;
966 if (MemFlags & SPIRV::CLK_IMAGE_MEM_FENCE)
967 MemSemantics |= SPIRV::MemorySemantics::ImageMemory;
969 if (Opcode == SPIRV::OpMemoryBarrier)
973 else if (Opcode == SPIRV::OpControlBarrierArriveINTEL)
974 MemSemantics |= SPIRV::MemorySemantics::Release;
975 else if (Opcode == SPIRV::OpControlBarrierWaitINTEL)
976 MemSemantics |= SPIRV::MemorySemantics::Acquire;
978 MemSemantics |= SPIRV::MemorySemantics::SequentiallyConsistent;
981 MemFlags == MemSemantics
985 SPIRV::Scope::Scope Scope = SPIRV::Scope::Workgroup;
986 SPIRV::Scope::Scope MemScope = Scope;
987 if (
Call->Arguments.size() >= 2) {
989 ((Opcode != SPIRV::OpMemoryBarrier &&
Call->Arguments.size() == 2) ||
990 (Opcode == SPIRV::OpMemoryBarrier &&
Call->Arguments.size() == 3)) &&
991 "Extra args for explicitly scoped barrier");
992 Register ScopeArg = (Opcode == SPIRV::OpMemoryBarrier) ?
Call->Arguments[2]
993 :
Call->Arguments[1];
994 SPIRV::CLMemoryScope CLScope =
997 if (!(MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE) ||
998 (Opcode == SPIRV::OpMemoryBarrier))
1000 if (CLScope ==
static_cast<unsigned>(Scope))
1001 ScopeReg =
Call->Arguments[1];
1008 if (Opcode != SPIRV::OpMemoryBarrier)
1010 MIB.
addUse(MemSemanticsReg);
1022 if ((Opcode == SPIRV::OpBitFieldInsert ||
1023 Opcode == SPIRV::OpBitFieldSExtract ||
1024 Opcode == SPIRV::OpBitFieldUExtract || Opcode == SPIRV::OpBitReverse) &&
1025 !ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {
1026 std::string DiagMsg = std::string(Builtin->
Name) +
1027 ": the builtin requires the following SPIR-V "
1028 "extension: SPV_KHR_bit_instructions";
1033 if (
Call->isSpirvOp())
1040 for (
unsigned i = 0; i <
Call->Arguments.size(); ++i)
1052 if (
Call->isSpirvOp())
1069 if (
Call->isSpirvOp())
1076 for (
unsigned i = 0; i <
Call->Arguments.size(); ++i)
1088 if (
Call->isSpirvOp())
1094 for (
unsigned i = 0; i <
Call->Arguments.size(); ++i)
1104 case SPIRV::OpCommitReadPipe:
1105 case SPIRV::OpCommitWritePipe:
1107 case SPIRV::OpGroupCommitReadPipe:
1108 case SPIRV::OpGroupCommitWritePipe:
1109 case SPIRV::OpGroupReserveReadPipePackets:
1110 case SPIRV::OpGroupReserveWritePipePackets: {
1114 MRI->setRegClass(ScopeConstReg, &SPIRV::iIDRegClass);
1118 if (Opcode == SPIRV::OpGroupReserveReadPipePackets ||
1119 Opcode == SPIRV::OpGroupReserveWritePipePackets)
1123 MIB.
addUse(ScopeConstReg);
1124 for (
unsigned int i = 0; i <
Call->Arguments.size(); ++i)
1137 case SPIRV::Dim::DIM_1D:
1138 case SPIRV::Dim::DIM_Buffer:
1140 case SPIRV::Dim::DIM_2D:
1141 case SPIRV::Dim::DIM_Cube:
1142 case SPIRV::Dim::DIM_Rect:
1144 case SPIRV::Dim::DIM_3D:
1157 return arrayed ? numComps + 1 : numComps;
1170 SPIRV::lookupExtendedBuiltin(Builtin->
Name, Builtin->
Set)->Number;
1177 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2) &&
1178 (
Number == SPIRV::OpenCLExtInst::fmin_common ||
1179 Number == SPIRV::OpenCLExtInst::fmax_common)) {
1181 ? SPIRV::OpenCLExtInst::fmin
1182 : SPIRV::OpenCLExtInst::fmax;
1190 .
addImm(
static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1196 if (OrigNumber == SPIRV::OpenCLExtInst::fmin_common ||
1197 OrigNumber == SPIRV::OpenCLExtInst::fmax_common) {
1211 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1215 std::tie(CompareRegister, RelationType) =
1228 Call->ReturnType, GR);
1236 SPIRV::lookupGroupBuiltin(Builtin->
Name);
1239 if (
Call->isSpirvOp()) {
1242 if (GroupBuiltin->
Opcode ==
1243 SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL &&
1244 Call->Arguments.size() > 4)
1253 if (!
MI ||
MI->getOpcode() != TargetOpcode::G_CONSTANT)
1255 "Group Operation parameter must be an integer constant");
1256 uint64_t GrpOp =
MI->getOperand(1).getCImm()->getValue().getZExtValue();
1263 for (
unsigned i = 2; i <
Call->Arguments.size(); ++i)
1276 if (ArgInstruction->
getOpcode() == TargetOpcode::G_CONSTANT) {
1277 if (BoolRegType->
getOpcode() != SPIRV::OpTypeBool)
1281 if (BoolRegType->
getOpcode() == SPIRV::OpTypeInt) {
1283 MRI->setRegClass(Arg0, &SPIRV::iIDRegClass);
1290 }
else if (BoolRegType->
getOpcode() != SPIRV::OpTypeBool) {
1302 const bool HasBoolReturnTy =
1307 if (HasBoolReturnTy)
1308 std::tie(GroupResultRegister, GroupResultType) =
1311 auto Scope = Builtin->
Name.
starts_with(
"sub_group") ? SPIRV::Scope::Subgroup
1312 : SPIRV::Scope::Workgroup;
1316 if (GroupBuiltin->
Opcode == SPIRV::OpGroupBroadcast &&
1317 Call->Arguments.size() > 2) {
1325 if (!ElemType || ElemType->
getOpcode() != SPIRV::OpTypeInt)
1327 unsigned VecLen =
Call->Arguments.size() - 1;
1328 VecReg =
MRI->createGenericVirtualRegister(
1330 MRI->setRegClass(VecReg, &SPIRV::vIDRegClass);
1336 for (
unsigned i = 1; i <
Call->Arguments.size(); i++) {
1337 MIB.addUse(
Call->Arguments[i]);
1346 .
addDef(GroupResultRegister)
1352 if (
Call->Arguments.size() > 0) {
1353 MIB.addUse(Arg0.
isValid() ? Arg0 :
Call->Arguments[0]);
1358 for (
unsigned i = 1; i <
Call->Arguments.size(); i++)
1359 MIB.addUse(
Call->Arguments[i]);
1363 if (HasBoolReturnTy)
1365 Call->ReturnType, GR);
1376 SPIRV::lookupIntelSubgroupsBuiltin(Builtin->
Name);
1378 if (IntelSubgroups->
IsMedia &&
1379 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {
1380 std::string DiagMsg = std::string(Builtin->
Name) +
1381 ": the builtin requires the following SPIR-V "
1382 "extension: SPV_INTEL_media_block_io";
1384 }
else if (!IntelSubgroups->
IsMedia &&
1385 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
1386 std::string DiagMsg = std::string(Builtin->
Name) +
1387 ": the builtin requires the following SPIR-V "
1388 "extension: SPV_INTEL_subgroups";
1393 if (
Call->isSpirvOp()) {
1394 bool IsSet = OpCode != SPIRV::OpSubgroupBlockWriteINTEL &&
1395 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL &&
1396 OpCode != SPIRV::OpSubgroupImageMediaBlockWriteINTEL;
1402 if (IntelSubgroups->
IsBlock) {
1405 if (Arg0Type->getOpcode() == SPIRV::OpTypeImage) {
1411 case SPIRV::OpSubgroupBlockReadINTEL:
1412 OpCode = SPIRV::OpSubgroupImageBlockReadINTEL;
1414 case SPIRV::OpSubgroupBlockWriteINTEL:
1415 OpCode = SPIRV::OpSubgroupImageBlockWriteINTEL;
1438 for (
size_t i = 0; i <
Call->Arguments.size(); ++i)
1449 if (!ST->canUseExtension(
1450 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
1451 std::string DiagMsg = std::string(Builtin->
Name) +
1452 ": the builtin requires the following SPIR-V "
1453 "extension: SPV_KHR_uniform_group_instructions";
1457 SPIRV::lookupGroupUniformBuiltin(Builtin->
Name);
1467 if (!Const || Const->getOpcode() != TargetOpcode::G_CONSTANT)
1469 "expect a constant group operation for a uniform group instruction",
1472 if (!ConstOperand.
isCImm())
1482 MIB.addUse(ValueReg);
1493 if (!ST->canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock)) {
1494 std::string DiagMsg = std::string(Builtin->
Name) +
1495 ": the builtin requires the following SPIR-V "
1496 "extension: SPV_KHR_shader_clock";
1503 SPIRV::Scope::Scope ScopeArg =
1505 .
EndsWith(
"device", SPIRV::Scope::Scope::Device)
1506 .
EndsWith(
"work_group", SPIRV::Scope::Scope::Workgroup)
1507 .
EndsWith(
"sub_group", SPIRV::Scope::Scope::Subgroup);
1547 SPIRV::BuiltIn::BuiltIn BuiltinValue,
1550 const unsigned ResultWidth =
Call->ReturnType->getOperand(1).getImm();
1561 bool IsConstantIndex =
1562 IndexInstruction->getOpcode() == TargetOpcode::G_CONSTANT;
1568 if (PointerSize != ResultWidth) {
1569 DefaultReg =
MRI->createGenericVirtualRegister(
LLT::scalar(PointerSize));
1570 MRI->setRegClass(DefaultReg, &SPIRV::iIDRegClass);
1572 MIRBuilder.
getMF());
1573 ToTruncate = DefaultReg;
1577 MIRBuilder.
buildCopy(DefaultReg, NewRegister);
1586 if (!IsConstantIndex || PointerSize != ResultWidth) {
1587 Extracted =
MRI->createGenericVirtualRegister(
LLT::scalar(PointerSize));
1588 MRI->setRegClass(Extracted, &SPIRV::iIDRegClass);
1595 ExtractInst.
addUse(LoadedVector).
addUse(IndexRegister);
1598 if (!IsConstantIndex) {
1607 MRI->setRegClass(CompareRegister, &SPIRV::iIDRegClass);
1622 if (PointerSize != ResultWidth) {
1625 MRI->setRegClass(SelectionResult, &SPIRV::iIDRegClass);
1627 MIRBuilder.
getMF());
1630 MIRBuilder.
buildSelect(SelectionResult, CompareRegister, Extracted,
1632 ToTruncate = SelectionResult;
1634 ToTruncate = Extracted;
1638 if (PointerSize != ResultWidth)
1648 SPIRV::BuiltIn::BuiltIn
Value =
1649 SPIRV::lookupGetBuiltin(Builtin->
Name, Builtin->
Set)->
Value;
1651 if (
Value == SPIRV::BuiltIn::GlobalInvocationId)
1657 if (
Call->ReturnType->getOpcode() == SPIRV::OpTypeVector)
1664 LLType,
Call->ReturnRegister);
1673 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1676 case SPIRV::OpStore:
1678 case SPIRV::OpAtomicLoad:
1680 case SPIRV::OpAtomicStore:
1682 case SPIRV::OpAtomicCompareExchange:
1683 case SPIRV::OpAtomicCompareExchangeWeak:
1686 case SPIRV::OpAtomicIAdd:
1687 case SPIRV::OpAtomicISub:
1688 case SPIRV::OpAtomicOr:
1689 case SPIRV::OpAtomicXor:
1690 case SPIRV::OpAtomicAnd:
1691 case SPIRV::OpAtomicExchange:
1693 case SPIRV::OpMemoryBarrier:
1695 case SPIRV::OpAtomicFlagTestAndSet:
1696 case SPIRV::OpAtomicFlagClear:
1699 if (
Call->isSpirvOp())
1711 unsigned Opcode = SPIRV::lookupAtomicFloatingBuiltin(Builtin->
Name)->Opcode;
1714 case SPIRV::OpAtomicFAddEXT:
1715 case SPIRV::OpAtomicFMinEXT:
1716 case SPIRV::OpAtomicFMaxEXT:
1729 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1740 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1742 if (Opcode == SPIRV::OpGenericCastToPtrExplicit) {
1743 SPIRV::StorageClass::StorageClass ResSC =
1754 MIRBuilder.
buildInstr(TargetOpcode::G_ADDRSPACE_CAST)
1765 if (
Call->isSpirvOp())
1770 SPIRV::OpTypeVector;
1772 uint32_t OC = IsVec ? SPIRV::OpDot : SPIRV::OpFMulS;
1773 bool IsSwapReq =
false;
1778 (ST->canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
1782 SPIRV::lookupIntegerDotProductBuiltin(Builtin->
Name);
1792 bool IsFirstSigned = TypeStrs[0].trim()[0] !=
'u';
1793 bool IsSecondSigned = TypeStrs[1].trim()[0] !=
'u';
1795 if (
Call->BuiltinName ==
"dot") {
1796 if (IsFirstSigned && IsSecondSigned)
1798 else if (!IsFirstSigned && !IsSecondSigned)
1801 OC = SPIRV::OpSUDot;
1805 }
else if (
Call->BuiltinName ==
"dot_acc_sat") {
1806 if (IsFirstSigned && IsSecondSigned)
1807 OC = SPIRV::OpSDotAccSat;
1808 else if (!IsFirstSigned && !IsSecondSigned)
1809 OC = SPIRV::OpUDotAccSat;
1811 OC = SPIRV::OpSUDotAccSat;
1827 for (
size_t i = 2; i <
Call->Arguments.size(); ++i)
1830 for (
size_t i = 0; i <
Call->Arguments.size(); ++i)
1836 if (!IsVec && OC != SPIRV::OpFMulS)
1837 MIB.
addImm(SPIRV::PackedVectorFormat4x8Bit);
1846 SPIRV::BuiltIn::BuiltIn
Value =
1847 SPIRV::lookupGetBuiltin(Builtin->
Name, Builtin->
Set)->
Value;
1850 assert(
Call->ReturnType->getOpcode() == SPIRV::OpTypeInt);
1854 MIRBuilder,
Call->ReturnType, GR,
Value, LLType,
Call->ReturnRegister,
1855 false, std::nullopt);
1869 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1876 if (RetType->
getOpcode() != SPIRV::OpTypeStruct)
1878 "overflow builtins");
1882 if (!OpType1 || !OpType2 || OpType1 != OpType2)
1884 if (OpType1->
getOpcode() == SPIRV::OpTypeVector)
1886 case SPIRV::OpIAddCarryS:
1887 Opcode = SPIRV::OpIAddCarryV;
1889 case SPIRV::OpISubBorrowS:
1890 Opcode = SPIRV::OpISubBorrowV;
1895 Register ResReg =
MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1897 MRI->getRegClassOrNull(
Call->Arguments[1])) {
1898 MRI->setRegClass(ResReg, DstRC);
1899 MRI->setType(ResReg,
MRI->getType(
Call->Arguments[1]));
1917 SPIRV::BuiltIn::BuiltIn
Value =
1918 SPIRV::lookupGetBuiltin(
Call->Builtin->Name,
Call->Builtin->Set)->
Value;
1919 const bool IsDefaultOne = (
Value == SPIRV::BuiltIn::GlobalSize ||
1920 Value == SPIRV::BuiltIn::NumWorkgroups ||
1921 Value == SPIRV::BuiltIn::WorkgroupSize ||
1922 Value == SPIRV::BuiltIn::EnqueuedWorkgroupSize);
1932 SPIRV::lookupImageQueryBuiltin(Builtin->
Name, Builtin->
Set)->Component;
1936 unsigned NumExpectedRetComponents =
1937 Call->ReturnType->getOpcode() == SPIRV::OpTypeVector
1938 ?
Call->ReturnType->getOperand(2).getImm()
1945 if (NumExpectedRetComponents != NumActualRetComponents) {
1946 unsigned Bitwidth =
Call->ReturnType->getOpcode() == SPIRV::OpTypeInt
1947 ?
Call->ReturnType->getOperand(1).getImm()
1954 IntTy, NumActualRetComponents, MIRBuilder,
true);
1959 IsDimBuf ? SPIRV::OpImageQuerySize : SPIRV::OpImageQuerySizeLod;
1966 if (NumExpectedRetComponents == NumActualRetComponents)
1968 if (NumExpectedRetComponents == 1) {
1970 unsigned ExtractedComposite =
1971 Component == 3 ? NumActualRetComponents - 1 : Component;
1972 assert(ExtractedComposite < NumActualRetComponents &&
1973 "Invalid composite index!");
1976 if (QueryResultType->
getOpcode() == SPIRV::OpTypeVector) {
1978 if (TypeReg != NewTypeReg &&
1980 TypeReg = NewTypeReg;
1982 MIRBuilder.
buildInstr(SPIRV::OpCompositeExtract)
1986 .
addImm(ExtractedComposite);
1987 if (NewType !=
nullptr)
1992 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpVectorShuffle)
1997 for (
unsigned i = 0; i < NumExpectedRetComponents; ++i)
1998 MIB.
addImm(i < NumActualRetComponents ? i : 0xffffffff);
2006 assert(
Call->ReturnType->getOpcode() == SPIRV::OpTypeInt &&
2007 "Image samples query result must be of int type!");
2012 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2015 SPIRV::Dim::Dim ImageDimensionality =
static_cast<SPIRV::Dim::Dim
>(
2017 (void)ImageDimensionality;
2020 case SPIRV::OpImageQuerySamples:
2021 assert(ImageDimensionality == SPIRV::Dim::DIM_2D &&
2022 "Image must be of 2D dimensionality");
2024 case SPIRV::OpImageQueryLevels:
2025 assert((ImageDimensionality == SPIRV::Dim::DIM_1D ||
2026 ImageDimensionality == SPIRV::Dim::DIM_2D ||
2027 ImageDimensionality == SPIRV::Dim::DIM_3D ||
2028 ImageDimensionality == SPIRV::Dim::DIM_Cube) &&
2029 "Image must be of 1D/2D/3D/Cube dimensionality");
2041static SPIRV::SamplerAddressingMode::SamplerAddressingMode
2043 switch (Bitmask & SPIRV::CLK_ADDRESS_MODE_MASK) {
2044 case SPIRV::CLK_ADDRESS_CLAMP:
2045 return SPIRV::SamplerAddressingMode::Clamp;
2046 case SPIRV::CLK_ADDRESS_CLAMP_TO_EDGE:
2047 return SPIRV::SamplerAddressingMode::ClampToEdge;
2048 case SPIRV::CLK_ADDRESS_REPEAT:
2049 return SPIRV::SamplerAddressingMode::Repeat;
2050 case SPIRV::CLK_ADDRESS_MIRRORED_REPEAT:
2051 return SPIRV::SamplerAddressingMode::RepeatMirrored;
2052 case SPIRV::CLK_ADDRESS_NONE:
2053 return SPIRV::SamplerAddressingMode::None;
2060 return (Bitmask & SPIRV::CLK_NORMALIZED_COORDS_TRUE) ? 1 : 0;
2063static SPIRV::SamplerFilterMode::SamplerFilterMode
2065 if (Bitmask & SPIRV::CLK_FILTER_LINEAR)
2066 return SPIRV::SamplerFilterMode::Linear;
2067 if (Bitmask & SPIRV::CLK_FILTER_NEAREST)
2068 return SPIRV::SamplerFilterMode::Nearest;
2069 return SPIRV::SamplerFilterMode::Nearest;
2076 if (
Call->isSpirvOp())
2083 if (HasOclSampler) {
2097 Register SampledImage =
MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2108 if (
Call->ReturnType->getOpcode() != SPIRV::OpTypeVector) {
2112 MRI->createGenericVirtualRegister(GR->
getRegType(TempType));
2115 MIRBuilder.
buildInstr(SPIRV::OpImageSampleExplicitLod)
2120 .
addImm(SPIRV::ImageOperand::Lod)
2122 MIRBuilder.
buildInstr(SPIRV::OpCompositeExtract)
2128 MIRBuilder.
buildInstr(SPIRV::OpImageSampleExplicitLod)
2133 .
addImm(SPIRV::ImageOperand::Lod)
2136 }
else if (HasMsaa) {
2142 .
addImm(SPIRV::ImageOperand::Sample)
2157 if (
Call->isSpirvOp())
2172 if (
Call->Builtin->Name.contains_insensitive(
2173 "__translate_sampler_initializer")) {
2180 return Sampler.isValid();
2181 }
else if (
Call->Builtin->Name.contains_insensitive(
"__spirv_SampledImage")) {
2188 Call->ReturnRegister.isValid()
2189 ?
Call->ReturnRegister
2190 :
MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2197 }
else if (
Call->Builtin->Name.contains_insensitive(
2198 "__spirv_ImageSampleExplicitLod")) {
2200 std::string ReturnType = DemangledCall.
str();
2201 if (DemangledCall.
contains(
"_R")) {
2202 ReturnType = ReturnType.substr(ReturnType.find(
"_R") + 2);
2203 ReturnType = ReturnType.substr(0, ReturnType.find(
'('));
2210 std::string DiagMsg =
2211 "Unable to recognize SPIRV type name: " + ReturnType;
2214 MIRBuilder.
buildInstr(SPIRV::OpImageSampleExplicitLod)
2219 .
addImm(SPIRV::ImageOperand::Lod)
2229 Call->Arguments[1],
Call->Arguments[2]);
2237 SPIRV::OpCompositeConstructContinuedINTEL,
2238 Call->Arguments,
Call->ReturnRegister,
2248 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2249 bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&
2250 Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&
2251 Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
2252 unsigned ArgSz =
Call->Arguments.size();
2253 unsigned LiteralIdx = 0;
2256 case SPIRV::OpCooperativeMatrixLoadKHR:
2257 LiteralIdx = ArgSz > 3 ? 3 : 0;
2259 case SPIRV::OpCooperativeMatrixStoreKHR:
2260 LiteralIdx = ArgSz > 4 ? 4 : 0;
2262 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2263 LiteralIdx = ArgSz > 7 ? 7 : 0;
2265 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2266 LiteralIdx = ArgSz > 8 ? 8 : 0;
2269 case SPIRV::OpCooperativeMatrixMulAddKHR:
2270 LiteralIdx = ArgSz > 3 ? 3 : 0;
2276 if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2278 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)
2295 if (Opcode == SPIRV::OpCooperativeMatrixLengthKHR) {
2306 IsSet ? TypeReg :
Register(0), ImmArgs);
2315 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2319 case SPIRV::OpSpecConstant: {
2329 (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
2330 Const->getOpcode() == TargetOpcode::G_FCONSTANT) &&
2331 "Argument should be either an int or floating-point constant");
2334 if (
Call->ReturnType->getOpcode() == SPIRV::OpTypeBool) {
2335 assert(ConstOperand.
isCImm() &&
"Int constant operand is expected");
2337 ? SPIRV::OpSpecConstantTrue
2338 : SPIRV::OpSpecConstantFalse;
2344 if (
Call->ReturnType->getOpcode() != SPIRV::OpTypeBool) {
2345 if (Const->getOpcode() == TargetOpcode::G_CONSTANT)
2352 case SPIRV::OpSpecConstantComposite: {
2354 SPIRV::OpSpecConstantCompositeContinuedINTEL,
2355 Call->Arguments,
Call->ReturnRegister,
2370 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2381 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2393 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2404 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2414 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2416 unsigned Scope = SPIRV::Scope::Workgroup;
2418 Scope = SPIRV::Scope::Subgroup;
2428 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2430 bool IsSet = Opcode != SPIRV::OpPredicatedStoreINTEL;
2431 unsigned ArgSz =
Call->Arguments.size();
2441 IsSet ? TypeReg :
Register(0), ImmArgs);
2454 Register TmpReg =
MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2458 unsigned NumArgs =
Call->Arguments.size();
2460 Register GlobalWorkSize =
Call->Arguments[NumArgs < 4 ? 1 : 2];
2462 NumArgs == 2 ?
Register(0) :
Call->Arguments[NumArgs < 4 ? 2 : 3];
2467 if (SpvTy->
getOpcode() == SPIRV::OpTypePointer) {
2473 unsigned Size =
Call->Builtin->Name ==
"ndrange_3D" ? 3 : 2;
2478 FieldTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
true);
2479 GlobalWorkSize =
MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2488 SpvFieldTy, *ST.getInstrInfo());
2493 LocalWorkSize = Const;
2494 if (!GlobalWorkOffset.
isValid())
2495 GlobalWorkOffset = Const;
2503 .
addUse(GlobalWorkOffset);
2517 SPIRV::AccessQualifier::ReadWrite,
true);
2525 bool IsSpirvOp =
Call->isSpirvOp();
2526 bool HasEvents =
Call->Builtin->Name.contains(
"events") || IsSpirvOp;
2533 if (
Call->Builtin->Name.contains(
"_varargs") || IsSpirvOp) {
2534 const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;
2542 assert(LocalSizeTy &&
"Local size type is expected");
2548 Int32Ty, MIRBuilder, SPIRV::StorageClass::Function);
2549 for (
unsigned I = 0;
I < LocalSizeNum; ++
I) {
2551 MRI->setType(
Reg, LLType);
2565 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpEnqueueKernel)
2570 const unsigned BlockFIdx = HasEvents ? 6 : 3;
2571 for (
unsigned i = 0; i < BlockFIdx; i++)
2572 MIB.addUse(
Call->Arguments[i]);
2579 MIB.addUse(NullPtr);
2580 MIB.addUse(NullPtr);
2588 Register BlockLiteralReg =
Call->Arguments[BlockFIdx + 1];
2590 MIB.addUse(BlockLiteralReg);
2600 for (
unsigned i = 0; i < LocalSizes.
size(); i++)
2601 MIB.addUse(LocalSizes[i]);
2611 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2614 case SPIRV::OpRetainEvent:
2615 case SPIRV::OpReleaseEvent:
2617 case SPIRV::OpCreateUserEvent:
2618 case SPIRV::OpGetDefaultQueue:
2622 case SPIRV::OpIsValidEvent:
2627 case SPIRV::OpSetUserEventStatus:
2631 case SPIRV::OpCaptureEventProfilingInfo:
2636 case SPIRV::OpBuildNDRange:
2638 case SPIRV::OpEnqueueKernel:
2651 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2653 bool IsSet = Opcode == SPIRV::OpGroupAsyncCopy;
2655 if (
Call->isSpirvOp())
2662 case SPIRV::OpGroupAsyncCopy: {
2664 Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent
2668 unsigned NumArgs =
Call->Arguments.size();
2678 ?
Call->Arguments[3]
2681 if (NewType !=
nullptr)
2686 case SPIRV::OpGroupWaitEvents:
2702 SPIRV::lookupConvertBuiltin(
Call->Builtin->Name,
Call->Builtin->Set);
2704 if (!Builtin &&
Call->isSpirvOp()) {
2707 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2712 assert(Builtin &&
"Conversion builtin not found.");
2715 SPIRV::Decoration::SaturatedConversion, {});
2718 SPIRV::Decoration::FPRoundingMode,
2719 {(unsigned)Builtin->RoundingMode});
2721 std::string NeedExtMsg;
2722 bool IsRightComponentsNumber =
true;
2723 unsigned Opcode = SPIRV::OpNop;
2730 : SPIRV::OpSatConvertSToU;
2733 : SPIRV::OpSConvert;
2735 SPIRV::OpTypeFloat)) {
2739 &MIRBuilder.
getMF().getSubtarget());
2740 if (!ST->canUseExtension(
2741 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
2742 NeedExtMsg =
"SPV_INTEL_bfloat16_conversion";
2743 IsRightComponentsNumber =
2746 Opcode = SPIRV::OpConvertBF16ToFINTEL;
2748 bool IsSourceSigned =
2750 Opcode = IsSourceSigned ? SPIRV::OpConvertSToF : SPIRV::OpConvertUToF;
2754 SPIRV::OpTypeFloat)) {
2760 &MIRBuilder.
getMF().getSubtarget());
2761 if (!ST->canUseExtension(
2762 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
2763 NeedExtMsg =
"SPV_INTEL_bfloat16_conversion";
2764 IsRightComponentsNumber =
2767 Opcode = SPIRV::OpConvertFToBF16INTEL;
2770 : SPIRV::OpConvertFToU;
2773 SPIRV::OpTypeFloat)) {
2776 &MIRBuilder.
getMF().getSubtarget());
2777 if (!ST->canUseExtension(
2778 SPIRV::Extension::SPV_INTEL_tensor_float32_conversion))
2779 NeedExtMsg =
"SPV_INTEL_tensor_float32_conversion";
2780 IsRightComponentsNumber =
2783 Opcode = SPIRV::OpRoundFToTF32INTEL;
2786 Opcode = SPIRV::OpFConvert;
2791 if (!NeedExtMsg.empty()) {
2792 std::string DiagMsg = std::string(Builtin->
Name) +
2793 ": the builtin requires the following SPIR-V "
2798 if (!IsRightComponentsNumber) {
2799 std::string DiagMsg =
2800 std::string(Builtin->
Name) +
2801 ": result and argument must have the same number of components";
2804 assert(Opcode != SPIRV::OpNop &&
2805 "Conversion between the types not implemented!");
2819 SPIRV::lookupVectorLoadStoreBuiltin(
Call->Builtin->Name,
2820 Call->Builtin->Set);
2826 .
addImm(
static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
2846 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2847 bool IsLoad = Opcode == SPIRV::OpLoad;
2851 MIB.addDef(
Call->ReturnRegister);
2859 MIB.addUse(
Call->Arguments[1]);
2861 unsigned NumArgs =
Call->Arguments.size();
2862 if ((IsLoad && NumArgs >= 2) || NumArgs >= 3)
2864 if ((IsLoad && NumArgs >= 3) || NumArgs >= 4)
2877std::tuple<int, unsigned, unsigned>
2879 SPIRV::InstructionSet::InstructionSet Set) {
2882 std::unique_ptr<const IncomingCall>
Call =
2885 return std::make_tuple(-1, 0, 0);
2887 switch (
Call->Builtin->Group) {
2888 case SPIRV::Relational:
2890 case SPIRV::Barrier:
2891 case SPIRV::CastToPtr:
2892 case SPIRV::ImageMiscQuery:
2893 case SPIRV::SpecConstant:
2894 case SPIRV::Enqueue:
2895 case SPIRV::AsyncCopy:
2896 case SPIRV::LoadStore:
2897 case SPIRV::CoopMatr:
2899 SPIRV::lookupNativeBuiltin(
Call->Builtin->Name,
Call->Builtin->Set))
2900 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2902 case SPIRV::Extended:
2903 if (
const auto *R = SPIRV::lookupExtendedBuiltin(
Call->Builtin->Name,
2904 Call->Builtin->Set))
2905 return std::make_tuple(
Call->Builtin->Group, 0, R->Number);
2907 case SPIRV::VectorLoadStore:
2908 if (
const auto *R = SPIRV::lookupVectorLoadStoreBuiltin(
Call->Builtin->Name,
2909 Call->Builtin->Set))
2910 return std::make_tuple(SPIRV::Extended, 0, R->Number);
2913 if (
const auto *R = SPIRV::lookupGroupBuiltin(
Call->Builtin->Name))
2914 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2916 case SPIRV::AtomicFloating:
2917 if (
const auto *R = SPIRV::lookupAtomicFloatingBuiltin(
Call->Builtin->Name))
2918 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2920 case SPIRV::IntelSubgroups:
2921 if (
const auto *R = SPIRV::lookupIntelSubgroupsBuiltin(
Call->Builtin->Name))
2922 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2924 case SPIRV::GroupUniform:
2925 if (
const auto *R = SPIRV::lookupGroupUniformBuiltin(
Call->Builtin->Name))
2926 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2928 case SPIRV::IntegerDot:
2930 SPIRV::lookupIntegerDotProductBuiltin(
Call->Builtin->Name))
2931 return std::make_tuple(
Call->Builtin->Group, R->Opcode, 0);
2933 case SPIRV::WriteImage:
2934 return std::make_tuple(
Call->Builtin->Group, SPIRV::OpImageWrite, 0);
2936 return std::make_tuple(
Call->Builtin->Group, TargetOpcode::G_SELECT, 0);
2937 case SPIRV::Construct:
2938 return std::make_tuple(
Call->Builtin->Group, SPIRV::OpCompositeConstruct,
2940 case SPIRV::KernelClock:
2941 return std::make_tuple(
Call->Builtin->Group, SPIRV::OpReadClockKHR, 0);
2943 return std::make_tuple(-1, 0, 0);
2945 return std::make_tuple(-1, 0, 0);
2949 SPIRV::InstructionSet::InstructionSet Set,
2954 LLVM_DEBUG(
dbgs() <<
"Lowering builtin call: " << DemangledCall <<
"\n");
2958 assert(SpvType &&
"Inconsistent return register: expected valid type info");
2959 std::unique_ptr<const IncomingCall>
Call =
2964 return std::nullopt;
2968 assert(Args.size() >=
Call->Builtin->MinNumArgs &&
2969 "Too few arguments to generate the builtin");
2970 if (
Call->Builtin->MaxNumArgs && Args.size() >
Call->Builtin->MaxNumArgs)
2971 LLVM_DEBUG(
dbgs() <<
"More arguments provided than required!\n");
2974 switch (
Call->Builtin->Group) {
2975 case SPIRV::Extended:
2977 case SPIRV::Relational:
2981 case SPIRV::Variable:
2985 case SPIRV::AtomicFloating:
2987 case SPIRV::Barrier:
2989 case SPIRV::CastToPtr:
2992 case SPIRV::IntegerDot:
2996 case SPIRV::ICarryBorrow:
2998 case SPIRV::GetQuery:
3000 case SPIRV::ImageSizeQuery:
3002 case SPIRV::ImageMiscQuery:
3004 case SPIRV::ReadImage:
3006 case SPIRV::WriteImage:
3008 case SPIRV::SampleImage:
3012 case SPIRV::Construct:
3014 case SPIRV::SpecConstant:
3016 case SPIRV::Enqueue:
3018 case SPIRV::AsyncCopy:
3020 case SPIRV::Convert:
3022 case SPIRV::VectorLoadStore:
3024 case SPIRV::LoadStore:
3026 case SPIRV::IntelSubgroups:
3028 case SPIRV::GroupUniform:
3030 case SPIRV::KernelClock:
3032 case SPIRV::CoopMatr:
3034 case SPIRV::ExtendedBitOps:
3036 case SPIRV::BindlessINTEL:
3038 case SPIRV::TernaryBitwiseINTEL:
3040 case SPIRV::Block2DLoadStore:
3044 case SPIRV::PredicatedLoadStore:
3055 [[maybe_unused]]
bool IsOCLBuiltinType = TypeStr.
consume_front(
"ocl_");
3056 assert(IsOCLBuiltinType &&
"Invalid OpenCL builtin prefix");
3073 unsigned VecElts = 0;
3084 TypeStr = TypeStr.
substr(0, TypeStr.
find(
']'));
3096 auto Pos1 = DemangledCall.
find(
'(');
3099 auto Pos2 = DemangledCall.
find(
')');
3102 DemangledCall.
slice(Pos1 + 1, Pos2)
3103 .
split(BuiltinArgsTypeStrs,
',', -1,
false);
3111 if (ArgIdx >= BuiltinArgsTypeStrs.
size())
3113 StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();
3122#define GET_BuiltinTypes_DECL
3123#define GET_BuiltinTypes_IMPL
3130#define GET_OpenCLTypes_DECL
3131#define GET_OpenCLTypes_IMPL
3133#include "SPIRVGenTables.inc"
3141 if (Name.starts_with(
"void"))
3143 else if (Name.starts_with(
"int") || Name.starts_with(
"uint"))
3145 else if (Name.starts_with(
"float"))
3147 else if (Name.starts_with(
"half"))
3160 unsigned Opcode = TypeRecord->
Opcode;
3175 "Invalid number of parameters for SPIR-V pipe builtin!");
3178 SPIRV::AccessQualifier::AccessQualifier(
3186 "Invalid number of parameters for SPIR-V coop matrices builtin!");
3188 "SPIR-V coop matrices builtin type must have a type parameter!");
3191 SPIRV::AccessQualifier::ReadWrite,
true);
3194 MIRBuilder, ExtensionType, ElemType, ExtensionType->
getIntParameter(0),
3203 OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder);
3212 "Inline SPIR-V type builtin takes an opcode, size, and alignment "
3219 if (ParamEType->getName() ==
"spirv.IntegralConstant") {
3220 assert(ParamEType->getNumTypeParameters() == 1 &&
3221 "Inline SPIR-V integral constant builtin must have a type "
3223 assert(ParamEType->getNumIntParameters() == 1 &&
3224 "Inline SPIR-V integral constant builtin must have a "
3227 auto OperandValue = ParamEType->getIntParameter(0);
3228 auto *OperandType = ParamEType->getTypeParameter(0);
3231 OperandType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
true);
3234 OperandValue, MIRBuilder, OperandSPIRVType,
true)));
3236 }
else if (ParamEType->getName() ==
"spirv.Literal") {
3237 assert(ParamEType->getNumTypeParameters() == 0 &&
3238 "Inline SPIR-V literal builtin does not take type "
3240 assert(ParamEType->getNumIntParameters() == 1 &&
3241 "Inline SPIR-V literal builtin must have an integer "
3244 auto OperandValue = ParamEType->getIntParameter(0);
3251 Param, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
true);
3263 "Vulkan buffers have exactly one type for the type of the buffer.");
3265 "Vulkan buffer have 2 integer parameters: storage class and is "
3269 auto SC =
static_cast<SPIRV::StorageClass::StorageClass
>(
3284 StringRef NameWithParameters = TypeName;
3291 SPIRV::lookupOpenCLType(NameWithParameters);
3294 NameWithParameters);
3302 "Unknown builtin opaque type!");
3306 if (!NameWithParameters.
contains(
'_'))
3310 unsigned BaseNameLength = NameWithParameters.
find(
'_') - 1;
3314 bool HasTypeParameter = !
isDigit(Parameters[0][0]);
3315 if (HasTypeParameter)
3318 for (
unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {
3319 unsigned IntParameter = 0;
3320 bool ValidLiteral = !Parameters[i].getAsInteger(10, IntParameter);
3323 "Invalid format of SPIR-V builtin parameter literal!");
3327 NameWithParameters.
substr(0, BaseNameLength),
3328 TypeParameters, IntParameters);
3332 SPIRV::AccessQualifier::AccessQualifier AccessQual,
3354 if (Name ==
"spirv.Type") {
3356 }
else if (Name ==
"spirv.VulkanBuffer") {
3358 }
else if (Name ==
"spirv.Layout") {
3372 switch (TypeRecord->
Opcode) {
3373 case SPIRV::OpTypeImage:
3376 case SPIRV::OpTypePipe:
3379 case SPIRV::OpTypeDeviceEvent:
3382 case SPIRV::OpTypeSampler:
3385 case SPIRV::OpTypeSampledImage:
3388 case SPIRV::OpTypeCooperativeMatrixKHR:
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Lower Kernel Arguments
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Promote Memory to Register
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
static const fltSemantics & IEEEsingle()
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
@ ICMP_ULT
unsigned less than
const APFloat & getValueAPF() const
const APInt & getValue() const
Return the constant as an APInt value reference.
A parsed version of the target data layout string in and methods for querying it.
Tagged union holding either a T or a Error.
Class to represent fixed width SIMD vectors.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
This is an important class for using LLVM in a threaded context.
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
LLVMContext & getContext() const
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
LLVM_ABI void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const MDNode * getMetadata() const
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
SPIRVType * getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccQual)
SPIRVType * getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVType * getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg, const MachineFunction &MF)
Register buildGlobalVariable(Register Reg, SPIRVType *BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
const Type * getTypeForSPIRVType(const SPIRVType *Ty) const
SPIRVType * getOrCreateUnknownType(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode, const ArrayRef< MCOperand > Operands)
unsigned getScalarOrVectorComponentCount(Register VReg) const
SPIRVType * getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
unsigned getPointerSize() const
SPIRVType * getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC)
SPIRVType * getOrCreateOpTypeByOpcode(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode)
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType=nullptr)
SPIRVType * getPointeeType(SPIRVType *PtrType)
Register getSPIRVTypeID(const SPIRVType *SpirvType) const
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
SPIRVType * getOrCreateOpTypeSampledImage(SPIRVType *ImageType, MachineIRBuilder &MIRBuilder)
SPIRVType * getOrCreateVulkanBufferType(MachineIRBuilder &MIRBuilder, Type *ElemType, SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr=false)
SPIRVType * getOrCreateSPIRVTypeByName(StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR, SPIRV::StorageClass::StorageClass SC=SPIRV::StorageClass::Function, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite)
SPIRVType * getOrCreateLayoutType(MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr=false)
Register getOrCreateConsIntVector(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR)
const TargetRegisterClass * getRegClass(SPIRVType *SpvType) const
SPIRVType * getOrCreateSPIRVVectorType(SPIRVType *BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
SPIRVType * getOrCreateOpTypeCoopMatr(MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType, const SPIRVType *ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns, uint32_t Use, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVType *SpvType, const SPIRVInstrInfo &TII)
SPIRVType * getOrCreateOpTypeDeviceEvent(MachineIRBuilder &MIRBuilder)
SPIRVType * getImageType(const TargetExtType *ExtensionType, const SPIRV::AccessQualifier::AccessQualifier Qualifier, MachineIRBuilder &MIRBuilder)
SPIRVType * getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR, bool ZeroAsNull=true)
LLT getRegType(SPIRVType *SpvType) const
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
SPIRVType * getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder)
Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param, unsigned FilerMode, MachineIRBuilder &MIRBuilder)
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVType *SpvType)
unsigned getScalarOrVectorBitWidth(const SPIRVType *Type) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
std::string str() const
str - Get the contents as an std::string.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
bool contains_insensitive(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
constexpr size_t size() const
size - Get the string size.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & EndsWith(StringLiteral S, T Value)
Class to represent struct types.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
ArrayRef< Type * > type_params() const
Return the type parameters for this particular target extension type.
unsigned getNumIntParameters() const
static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
Type * getTypeParameter(unsigned i) const
unsigned getNumTypeParameters() const
unsigned getIntParameter(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM_ABI StringRef getStructName() const
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
LLVM Value Representation.
LLVM_ABI Value(Type *Ty, unsigned scid)
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Represents a version number in the form major[.minor[.subminor[.build]]].
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
LLVM_C_ABI LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount)
Create a vector type that contains a defined type and has a specific number of elements.
std::string lookupBuiltinNameHelper(StringRef DemangledCall, FPDecorationId *DecorationId)
Parses the name part of the demangled builtin call.
Type * parseBuiltinCallArgumentType(StringRef TypeStr, LLVMContext &Ctx)
bool parseBuiltinTypeStr(SmallVector< StringRef, 10 > &BuiltinArgsTypeStrs, const StringRef DemangledCall, LLVMContext &Ctx)
std::optional< bool > lowerBuiltin(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, MachineIRBuilder &MIRBuilder, const Register OrigRet, const Type *OrigRetTy, const SmallVectorImpl< Register > &Args, SPIRVGlobalRegistry *GR, const CallBase &CB)
std::tuple< int, unsigned, unsigned > mapBuiltinToOpcode(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set)
Helper function for finding a builtin function attributes by a demangled function name.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
TargetExtType * parseBuiltinTypeNameToTargetExtType(std::string TypeName, LLVMContext &Context)
Translates a string representing a SPIR-V or OpenCL builtin type to a TargetExtType that can be furth...
SPIRVType * lowerBuiltinType(const Type *OpaqueType, SPIRV::AccessQualifier::AccessQualifier AccessQual, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
This is an optimization pass for GlobalISel generic memory operations.
void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)
static bool build2DBlockIOINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building Intel's 2d block io instructions.
static SPIRVType * getVulkanBufferType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateExtInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, const CallBase &CB)
void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
Helper external function for inserting ASSIGN_TYPE instuction between Reg and its definition,...
static bool generateBindlessImageINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateGetQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getInlineSpirvType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateConstructInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building atomic flag instructions (e.g.
static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRV::SamplerFilterMode::SamplerFilterMode getSamplerFilterModeFromBitmask(unsigned Bitmask)
static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic store instruction.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
static bool buildExtendedBitOpsInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building extended bit operations.
static const Type * getBlockStructType(Register ParamReg, MachineRegisterInfo *MRI)
static bool generateGroupInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
FPDecorationId demangledPostfixToDecorationId(const std::string &S)
static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim)
static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Register buildScopeReg(Register CLScopeRegister, SPIRV::Scope::Scope Scope, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI)
static std::tuple< Register, SPIRVType * > buildBoolRegister(MachineIRBuilder &MIRBuilder, const SPIRVType *ResultType, SPIRVGlobalRegistry *GR)
Helper function building either a resulting scalar or vector bool register depending on the expected ...
static unsigned getNumSizeComponents(SPIRVType *imgType)
Helper function for obtaining the number of size components.
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
static Register buildConstantIntReg32(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getSampledImageType(const TargetExtType *OpaqueType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
static bool generateSampleImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateBarrierInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
LLVM_ABI void SplitString(StringRef Source, SmallVectorImpl< StringRef > &OutFragments, StringRef Delimiters=" \t\n\v\f\r")
SplitString - Split up the specified string according to the specified delimiters,...
static SPIRVType * getCoopMatrType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateKernelClockInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static void setRegClassIfNull(Register Reg, MachineRegisterInfo *MRI, SPIRVGlobalRegistry *GR)
static bool generateGroupUniformInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateWaveInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)
Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building barriers, i.e., memory/control ordering operations.
static bool generateAsyncCopy(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRV::Scope::Scope getSPIRVScope(SPIRV::CLMemoryScope ClScope)
static SPIRVType * getSamplerType(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static Register buildLoadInst(SPIRVType *BaseType, Register PtrRegister, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, LLT LowLevelType, Register DestinationReg=Register(0))
Helper function for building a load instruction loading into the DestinationReg.
static bool generateEnqueueInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
static bool buildSelectInst(MachineIRBuilder &MIRBuilder, Register ReturnRegister, Register SourceRegister, const SPIRVType *ReturnType, SPIRVGlobalRegistry *GR)
Helper function for building either a vector or scalar select instruction depending on the expected R...
static const Type * getMachineInstrType(MachineInstr *MI)
bool isDigit(char C)
Checks if character C is one of the 10 decimal digits.
static SPIRV::SamplerAddressingMode::SamplerAddressingMode getSamplerAddressingModeFromBitmask(unsigned Bitmask)
static bool generateAtomicInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
const MachineInstr SPIRVType
static SPIRVType * getLayoutType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateDotOrFMulInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Register buildBuiltinVariableLoad(MachineIRBuilder &MIRBuilder, SPIRVType *VariableType, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType, Register Reg=Register(0), bool isConst=true, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageTy={ SPIRV::LinkageType::Import})
Helper function for building a load instruction for loading a builtin global variable of BuiltinValue...
static bool generateConvertInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateTernaryBitwiseFunctionINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
static Register buildMemSemanticsReg(Register SemanticsRegister, Register PtrRegister, unsigned &Semantics, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static unsigned getConstFromIntrinsic(Register Reg, MachineRegisterInfo *MRI)
static bool generateImageMiscQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateSelectInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)
static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic load instruction.
static bool generateIntelSubgroupsInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateExtendedBitOpsInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildPipeInst(const SPIRV::IncomingCall *Call, unsigned Opcode, unsigned Scope, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateSpecConstantInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getOrCreateSPIRVDeviceEventPointer(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)
static bool generateVectorLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, uint64_t DefaultValue)
static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildBindlessImageINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building Intel's bindless image instructions.
static std::unique_ptr< const SPIRV::IncomingCall > lookupBuiltin(StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)
Looks up the demangled builtin call in the SPIRVBuiltins.td records using the provided DemangledCall ...
static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic floating-type instruction.
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
static bool generate2DBlockIOINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateReadImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool hasBuiltinTypePrefix(StringRef Name)
static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Type * getMDOperandAsType(const MDNode *N, unsigned I)
static bool generatePipeInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildTernaryBitwiseFunctionINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building Intel's OpBitwiseFunctionINTEL instruction.
static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building atomic instructions.
static SPIRV::MemorySemantics::MemorySemantics getSPIRVMemSemantics(std::memory_order MemOrder)
static bool generateRelationalInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)
Helper function for translating atomic init to OpStore.
static bool generateWriteImageInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getPipeType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Type * parseTypeString(const StringRef Name, LLVMContext &Context)
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
static bool generatePredicatedLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateAtomicFloatingInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildNDRange(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getNonParameterizedType(const TargetExtType *ExtensionType, const SPIRV::BuiltinType *TypeRecord, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static MachineInstr * getBlockStructInstr(Register ParamReg, MachineRegisterInfo *MRI)
static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode, const SPIRV::IncomingCall *Call, Register TypeReg, ArrayRef< uint32_t > ImmArgs={})
static unsigned getSamplerParamFromBitmask(unsigned Bitmask)
static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic compare-exchange instruction.
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
static bool generateBuiltinVar(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Implement std::hash so that hash_code can be used in STL containers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
This class contains a discriminated union of information about pointers in memory operands,...
FPRoundingMode::FPRoundingMode RoundingMode
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
const SmallVectorImpl< Register > & Arguments
const std::string BuiltinName
const SPIRVType * ReturnType
const Register ReturnRegister
const DemangledBuiltin * Builtin
IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin, const Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)
InstructionSet::InstructionSet Set
StringRef SpirvTypeLiteral
InstructionSet::InstructionSet Set
FPRoundingMode::FPRoundingMode RoundingMode