23#include "llvm/IR/IntrinsicsSPIRV.h"
53class SPIRVEmitIntrinsics
55 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
59 bool TrackConstants =
true;
70 Type *deduceElementTypeHelper(
Value *
I, std::unordered_set<Value *> &Visited);
71 Type *deduceElementTypeByValueDeep(
Type *ValueTy,
Value *Operand,
72 std::unordered_set<Value *> &Visited);
74 std::unordered_set<Value *> &Visited);
77 Type *deduceNestedTypeHelper(
User *U);
79 std::unordered_set<Value *> &Visited);
96 for (
auto *Imm : Imms)
98 return B.CreateIntrinsic(IntrID, {
Types},
Args);
107 void insertAssignTypeInstrForTargetExtTypes(
TargetExtType *AssignedType,
110 Type *ExpectedElementType,
111 unsigned OperandToReplace,
118 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx);
119 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx,
120 std::unordered_set<Function *> &FVisited);
155char SPIRVEmitIntrinsics::ID = 0;
161 return isa<IntrinsicInst>(
I) &&
162 cast<IntrinsicInst>(
I)->getIntrinsicID() == Intrinsic::spv_assign_type;
166 return isa<StoreInst>(
I) || isa<LoadInst>(
I) || isa<InsertValueInst>(
I) ||
167 isa<ExtractValueInst>(
I) || isa<AtomicCmpXchgInst>(
I);
171 return isa<ConstantArray>(V) || isa<ConstantStruct>(V) ||
172 isa<ConstantDataArray>(V) ||
173 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
178 B.SetInsertPoint(
I->getParent()->getFirstNonPHIOrDbgOrAlloca());
186 switch (
Intr->getIntrinsicID()) {
187 case Intrinsic::invariant_start:
188 case Intrinsic::invariant_end:
196 if (
I->getType()->isTokenTy())
198 "does not support token type",
205 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Arg->
getType()},
210 AssignPtrTypeInstr[Arg] = AssignPtrTyCI;
215Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
216 Type *ValueTy,
Value *Operand, std::unordered_set<Value *> &Visited) {
219 if (
auto *PtrTy = dyn_cast<PointerType>(Ty)) {
220 if (
Type *NestedTy = deduceElementTypeHelper(Operand, Visited))
223 Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited);
230Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
231 Value *
Op, std::unordered_set<Value *> &Visited) {
235 if (
auto PType = dyn_cast<TypedPointerType>(
Op->getType()))
236 return PType->getElementType();
242 for (
User *OpU :
Op->users()) {
243 if (
Instruction *Inst = dyn_cast<Instruction>(OpU)) {
244 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
256 Function *CalledF,
unsigned OpIdx) {
257 if ((DemangledName.
starts_with(
"__spirv_ocl_printf(") ||
260 return IntegerType::getInt8Ty(CalledF->
getContext());
266Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
Value *
I) {
267 std::unordered_set<Value *> Visited;
268 return deduceElementTypeHelper(
I, Visited);
271Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
272 Value *
I, std::unordered_set<Value *> &Visited) {
282 if (Visited.find(
I) != Visited.end())
289 if (
auto *
Ref = dyn_cast<AllocaInst>(
I)) {
290 Ty =
Ref->getAllocatedType();
291 }
else if (
auto *
Ref = dyn_cast<GetElementPtrInst>(
I)) {
292 Ty =
Ref->getResultElementType();
293 }
else if (
auto *
Ref = dyn_cast<GlobalValue>(
I)) {
294 Ty = deduceElementTypeByValueDeep(
296 Ref->getNumOperands() > 0 ?
Ref->getOperand(0) :
nullptr, Visited);
297 }
else if (
auto *
Ref = dyn_cast<AddrSpaceCastInst>(
I)) {
298 Ty = deduceElementTypeHelper(
Ref->getPointerOperand(), Visited);
299 }
else if (
auto *
Ref = dyn_cast<BitCastInst>(
I)) {
300 if (
Type *Src =
Ref->getSrcTy(), *Dest =
Ref->getDestTy();
302 Ty = deduceElementTypeHelper(
Ref->getOperand(0), Visited);
303 }
else if (
auto *
Ref = dyn_cast<AtomicCmpXchgInst>(
I)) {
305 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
306 }
else if (
auto *
Ref = dyn_cast<AtomicRMWInst>(
I)) {
308 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
309 }
else if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
310 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
311 Ty = deduceElementTypeByUsersDeep(
Ref->getIncomingValue(i), Visited);
315 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
316 for (
Value *
Op : {
Ref->getTrueValue(),
Ref->getFalseValue()}) {
317 Ty = deduceElementTypeByUsersDeep(
Op, Visited);
335Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
User *U) {
336 std::unordered_set<Value *> Visited;
337 return deduceNestedTypeHelper(U,
U->getType(), Visited);
340Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
341 User *U,
Type *OrigTy, std::unordered_set<Value *> &Visited) {
350 if (Visited.find(U) != Visited.end())
354 if (dyn_cast<StructType>(OrigTy)) {
357 for (
unsigned i = 0; i <
U->getNumOperands(); ++i) {
359 Type *OpTy =
Op->getType();
362 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
363 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
366 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
370 Change |= Ty != OpTy;
377 }
else if (
auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) {
378 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
379 Type *OpTy = ArrTy->getElementType();
381 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
382 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
385 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
388 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
393 }
else if (
auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
394 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
395 Type *OpTy = VecTy->getElementType();
397 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
398 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
401 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
404 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
414Type *SPIRVEmitIntrinsics::deduceElementType(
Value *
I) {
415 if (
Type *Ty = deduceElementTypeHelper(
I))
417 return IntegerType::getInt8Ty(
I->getContext());
424void SPIRVEmitIntrinsics::deduceOperandElementType(
Instruction *
I) {
426 Type *KnownElemTy =
nullptr;
428 if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
432 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
437 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
441 for (
unsigned i = 0; i <
Ref->getNumOperands(); i++) {
446 }
else if (
auto *
Ref = dyn_cast<ReturnInst>(
I)) {
463 }
else if (
auto *
Ref = dyn_cast<ICmpInst>(
I)) {
471 KnownElemTy = ElemTy0;
473 }
else if (ElemTy1) {
474 KnownElemTy = ElemTy1;
480 if (!KnownElemTy || Ops.
size() == 0)
485 for (
auto &OpIt : Ops) {
490 if (Ty == KnownElemTy)
497 Type *OpTy =
Op->getType();
501 auto It = AssignPtrTypeInstr.
find(
Op);
502 if (It == AssignPtrTypeInstr.
end()) {
504 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal,
Op,
506 AssignPtrTypeInstr[
Op] = CI;
520 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
521 I->setOperand(OpIt.second, PtrCastI);
526void SPIRVEmitIntrinsics::replaceMemInstrUses(
Instruction *Old,
531 if (isAssignTypeInstr(U)) {
534 B.CreateIntrinsic(Intrinsic::spv_assign_type, {
New->getType()},
Args);
535 U->eraseFromParent();
538 U->replaceUsesOfWith(Old, New);
546void SPIRVEmitIntrinsics::preprocessUndefs(
IRBuilder<> &
B) {
547 std::queue<Instruction *> Worklist;
551 while (!Worklist.empty()) {
555 for (
auto &
Op :
I->operands()) {
556 auto *AggrUndef = dyn_cast<UndefValue>(
Op);
557 if (!AggrUndef || !
Op->getType()->isAggregateType())
561 auto *IntrUndef =
B.CreateIntrinsic(Intrinsic::spv_undef, {}, {});
562 Worklist.push(IntrUndef);
563 I->replaceUsesOfWith(
Op, IntrUndef);
564 AggrConsts[IntrUndef] = AggrUndef;
565 AggrConstTypes[IntrUndef] = AggrUndef->getType();
570void SPIRVEmitIntrinsics::preprocessCompositeConstants(
IRBuilder<> &
B) {
571 std::queue<Instruction *> Worklist;
575 while (!Worklist.empty()) {
576 auto *
I = Worklist.front();
578 bool KeepInst =
false;
579 for (
const auto &
Op :
I->operands()) {
581 Type *ResTy =
nullptr;
582 if (
auto *COp = dyn_cast<ConstantVector>(
Op)) {
583 AggrConst = cast<Constant>(COp);
584 ResTy = COp->getType();
585 }
else if (
auto *COp = dyn_cast<ConstantArray>(
Op)) {
586 AggrConst = cast<Constant>(COp);
587 ResTy =
B.getInt32Ty();
588 }
else if (
auto *COp = dyn_cast<ConstantStruct>(
Op)) {
589 AggrConst = cast<Constant>(COp);
590 ResTy =
B.getInt32Ty();
591 }
else if (
auto *COp = dyn_cast<ConstantDataArray>(
Op)) {
592 AggrConst = cast<Constant>(COp);
593 ResTy =
B.getInt32Ty();
594 }
else if (
auto *COp = dyn_cast<ConstantAggregateZero>(
Op)) {
595 if (!
Op->getType()->isVectorTy()) {
596 AggrConst = cast<Constant>(COp);
597 ResTy =
B.getInt32Ty();
602 if (
auto *COp = dyn_cast<ConstantDataSequential>(
Op))
603 for (
unsigned i = 0; i < COp->getNumElements(); ++i)
604 Args.push_back(COp->getElementAsConstant(i));
606 for (
auto &COp : AggrConst->
operands())
610 B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {
Args});
612 I->replaceUsesOfWith(
Op, CI);
614 AggrConsts[CI] = AggrConst;
615 AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst);
626 B.SetInsertPoint(&
I);
629 for (
auto &
Op :
I.operands()) {
630 if (
Op.get()->getType()->isSized()) {
632 }
else if (
BasicBlock *BB = dyn_cast<BasicBlock>(
Op.get())) {
639 CallInst *NewI =
B.CreateIntrinsic(Intrinsic::spv_switch,
643 I.replaceAllUsesWith(NewI);
647 B.SetInsertPoint(ParentBB);
658 B.SetInsertPoint(&
I);
661 Args.push_back(
B.getInt1(
I.isInBounds()));
662 for (
auto &
Op :
I.operands())
664 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_gep, {
Types}, {
Args});
665 I.replaceAllUsesWith(NewI);
672 B.SetInsertPoint(&
I);
680 I.replaceAllUsesWith(Source);
687 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_bitcast, {
Types}, {
Args});
688 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
689 I.replaceAllUsesWith(NewI);
695void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes(
698 if (
V->getType() == AssignedType)
704 for (
auto User :
V->users()) {
705 auto *II = dyn_cast<IntrinsicInst>(
User);
706 if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_type)
711 dyn_cast<ConstantAsMetadata>(VMD->
getMetadata())->getType();
712 if (BuiltinType != AssignedType)
715 " for value " +
V->getName(),
721 buildIntrWithMD(Intrinsic::spv_assign_type, {
V->getType()},
Const,
V, {},
B);
724void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
729 while (
BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
733 Type *PointerElemTy = deduceElementTypeHelper(Pointer);
734 if (PointerElemTy == ExpectedElementType)
738 Constant *ExpectedElementTypeConst =
745 bool FirstPtrCastOrAssignPtrType =
true;
751 auto *II = dyn_cast<IntrinsicInst>(
User);
753 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
754 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
755 II->getOperand(0) != Pointer)
760 FirstPtrCastOrAssignPtrType =
false;
761 if (II->getOperand(1) != VMD ||
762 dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() !=
768 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
773 if (II->getParent() !=
I->getParent())
776 I->setOperand(OperandToReplace, II);
787 if (FirstPtrCastOrAssignPtrType &&
788 (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
790 Intrinsic::spv_assign_ptr_type, {
Pointer->getType()},
794 AssignPtrTypeInstr[
Pointer] = CI;
801 auto *PtrCastI =
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
802 I->setOperand(OperandToReplace, PtrCastI);
805void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(
Instruction *
I,
811 isa<Argument>(
SI->getValueOperand())) {
812 return replacePointerOperandWithPtrCast(
813 I,
SI->getValueOperand(), IntegerType::getInt8Ty(
F->getContext()), 0,
816 return replacePointerOperandWithPtrCast(
817 I,
SI->getPointerOperand(),
SI->getValueOperand()->getType(), 1,
B);
818 }
else if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
819 return replacePointerOperandWithPtrCast(
I, LI->getPointerOperand(),
820 LI->getType(), 0,
B);
822 return replacePointerOperandWithPtrCast(
I, GEPI->getPointerOperand(),
823 GEPI->getSourceElementType(), 0,
B);
833 std::string DemangledName =
837 bool HaveTypes =
false;
838 for (
unsigned OpIdx = 0; OpIdx < CalledF->
arg_size(); ++OpIdx) {
844 CalledArgTys.
push_back(cast<TypedPointerType>(ArgType)->getElementType());
856 if (
Instruction *Inst = dyn_cast<Instruction>(U)) {
857 if ((ElemTy = deduceElementTypeHelper(Inst)) !=
nullptr)
863 HaveTypes |= ElemTy !=
nullptr;
868 if (DemangledName.empty() && !HaveTypes)
871 for (
unsigned OpIdx = 0; OpIdx < CI->
arg_size(); OpIdx++) {
873 if (!isa<PointerType>(ArgOperand->
getType()) &&
874 !isa<TypedPointerType>(ArgOperand->
getType()))
878 if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) {
888 OpIdx < CalledArgTys.
size() ? CalledArgTys[OpIdx] :
nullptr;
889 if (!ExpectedType && !DemangledName.empty())
891 DemangledName, OpIdx,
I->getContext());
896 insertAssignTypeInstrForTargetExtTypes(cast<TargetExtType>(ExpectedType),
899 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx,
B);
905 I.getOperand(1)->getType(),
906 I.getOperand(2)->getType()};
908 B.SetInsertPoint(&
I);
910 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_insertelt, {
Types}, {
Args});
911 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
912 I.replaceAllUsesWith(NewI);
921 B.SetInsertPoint(&
I);
923 I.getIndexOperand()->getType()};
925 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_extractelt, {
Types}, {
Args});
926 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
927 I.replaceAllUsesWith(NewI);
935 B.SetInsertPoint(&
I);
938 for (
auto &
Op :
I.operands())
939 if (isa<UndefValue>(
Op))
943 for (
auto &
Op :
I.indices())
944 Args.push_back(
B.getInt32(
Op));
946 B.CreateIntrinsic(Intrinsic::spv_insertv, {
Types}, {
Args});
947 replaceMemInstrUses(&
I, NewI,
B);
953 B.SetInsertPoint(&
I);
955 for (
auto &
Op :
I.operands())
957 for (
auto &
Op :
I.indices())
958 Args.push_back(
B.getInt32(
Op));
960 B.CreateIntrinsic(Intrinsic::spv_extractv, {
I.getType()}, {
Args});
961 I.replaceAllUsesWith(NewI);
967 if (!
I.getType()->isAggregateType())
970 B.SetInsertPoint(&
I);
971 TrackConstants =
false;
972 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
974 TLI->getLoadMemOperandFlags(
I,
F->getParent()->getDataLayout());
976 B.CreateIntrinsic(Intrinsic::spv_load, {
I.getOperand(0)->
getType()},
977 {
I.getPointerOperand(),
B.getInt16(Flags),
978 B.getInt8(
I.getAlign().value())});
979 replaceMemInstrUses(&
I, NewI,
B);
987 B.SetInsertPoint(&
I);
988 TrackConstants =
false;
989 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
991 TLI->getStoreMemOperandFlags(
I,
F->getParent()->getDataLayout());
992 auto *PtrOp =
I.getPointerOperand();
993 auto *NewI =
B.CreateIntrinsic(
994 Intrinsic::spv_store, {
I.getValueOperand()->
getType(), PtrOp->getType()},
995 {
I.getValueOperand(), PtrOp,
B.getInt16(Flags),
996 B.getInt8(
I.getAlign().value())});
1002 Value *ArraySize =
nullptr;
1003 if (
I.isArrayAllocation()) {
1006 SPIRV::Extension::SPV_INTEL_variable_length_array))
1008 "array allocation: this instruction requires the following "
1009 "SPIR-V extension: SPV_INTEL_variable_length_array",
1011 ArraySize =
I.getArraySize();
1014 B.SetInsertPoint(&
I);
1015 TrackConstants =
false;
1016 Type *PtrTy =
I.getType();
1018 ArraySize ?
B.CreateIntrinsic(Intrinsic::spv_alloca_array,
1019 {PtrTy, ArraySize->
getType()}, {ArraySize})
1020 :
B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
1021 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
1022 I.replaceAllUsesWith(NewI);
1023 I.eraseFromParent();
1029 assert(
I.getType()->isAggregateType() &&
"Aggregate result is expected");
1031 B.SetInsertPoint(&
I);
1033 for (
auto &
Op :
I.operands())
1035 Args.push_back(
B.getInt32(
I.getSyncScopeID()));
1036 Args.push_back(
B.getInt32(
1038 Args.push_back(
B.getInt32(
1040 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_cmpxchg,
1042 replaceMemInstrUses(&
I, NewI,
B);
1048 B.SetInsertPoint(&
I);
1049 B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
1056 if (GV.
getName() ==
"llvm.global.annotations")
1062 deduceElementTypeHelper(&GV);
1066 auto *InitInst =
B.CreateIntrinsic(Intrinsic::spv_init_global,
1068 InitInst->setArgOperand(1,
Init);
1072 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.
getType(), &GV);
1075void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(
Instruction *
I,
1079 isa<BitCastInst>(
I))
1084 Type *ElemTy = deduceElementType(
I);
1087 CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {
I->getType()},
1090 AssignPtrTypeInstr[
I] = CI;
1093void SPIRVEmitIntrinsics::insertAssignTypeIntrs(
Instruction *
I,
1096 Type *Ty =
I->getType();
1099 Type *TypeToAssign = Ty;
1100 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
1101 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
1102 II->getIntrinsicID() == Intrinsic::spv_undef) {
1103 auto It = AggrConstTypes.
find(II);
1104 if (It == AggrConstTypes.
end())
1106 TypeToAssign = It->second;
1110 buildIntrWithMD(Intrinsic::spv_assign_type, {Ty},
Const,
I, {},
B);
1112 for (
const auto &
Op :
I->operands()) {
1113 if (isa<ConstantPointerNull>(
Op) || isa<UndefValue>(
Op) ||
1115 (isa<ConstantExpr>(
Op) && isa<GEPOperator>(
Op))) {
1117 if (isa<UndefValue>(
Op) &&
Op->getType()->isAggregateType())
1118 buildIntrWithMD(Intrinsic::spv_assign_type, {
B.getInt32Ty()},
Op,
1120 else if (!isa<Instruction>(
Op))
1121 buildIntrWithMD(Intrinsic::spv_assign_type, {
Op->getType()},
Op,
Op, {},
1127void SPIRVEmitIntrinsics::insertSpirvDecorations(
Instruction *
I,
1129 if (
MDNode *MD =
I->getMetadata(
"spirv.Decorations")) {
1130 B.SetInsertPoint(
I->getNextNode());
1131 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
1136void SPIRVEmitIntrinsics::processInstrAfterVisit(
Instruction *
I,
1138 auto *II = dyn_cast<IntrinsicInst>(
I);
1139 if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
1141 B.SetInsertPoint(
I->getNextNode());
1142 auto t = AggrConsts.
find(
I);
1145 buildIntrWithMD(Intrinsic::spv_track_constant,
1146 {II->getType(), II->getType()}, t->second,
I, {},
B);
1147 I->replaceAllUsesWith(NewOp);
1148 NewOp->setArgOperand(0,
I);
1150 for (
const auto &
Op :
I->operands()) {
1151 if ((isa<ConstantAggregateZero>(
Op) &&
Op->getType()->isVectorTy()) ||
1152 isa<PHINode>(
I) || isa<SwitchInst>(
I))
1153 TrackConstants =
false;
1154 if ((isa<ConstantData>(
Op) || isa<ConstantExpr>(
Op)) && TrackConstants) {
1155 unsigned OpNo =
Op.getOperandNo();
1156 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
1157 (II->paramHasAttr(OpNo, Attribute::ImmArg))))
1159 B.SetInsertPoint(
I);
1161 if (
Op->getType()->isTargetExtTy())
1164 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
1167 I->setOperand(OpNo, NewOp);
1173 std::vector<Value *>
Args = {
I};
1175 B.CreateIntrinsic(Intrinsic::spv_assign_name, {
I->getType()},
Args);
1179Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
Function *
F,
1181 std::unordered_set<Function *> FVisited;
1182 return deduceFunParamElementType(
F, OpIdx, FVisited);
1185Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
1186 Function *
F,
unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
1188 if (FVisited.find(
F) != FVisited.end())
1192 std::unordered_set<Value *> Visited;
1195 for (
User *U :
F->users()) {
1196 CallInst *CI = dyn_cast<CallInst>(U);
1197 if (!CI || OpIdx >= CI->
arg_size())
1207 if (
Type *Ty = deduceElementTypeHelper(OpArg, Visited))
1212 if (!Inst || Inst == CI)
1215 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
1222 if (FVisited.find(OuterF) != FVisited.end())
1224 for (
unsigned i = 0; i < OuterF->
arg_size(); ++i) {
1225 if (OuterF->
getArg(i) == OpArg) {
1226 Lookup.push_back(std::make_pair(OuterF, i));
1233 for (
auto &Pair :
Lookup) {
1234 if (
Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
1241void SPIRVEmitIntrinsics::processParamTypesByFunHeader(
Function *
F,
1243 B.SetInsertPointPastAllocas(
F);
1244 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1251 buildAssignPtr(
B, ElemTy, Arg);
1256 B.SetInsertPointPastAllocas(
F);
1257 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1262 if (!ElemTy && (ElemTy = deduceFunParamElementType(
F, OpIdx)) !=
nullptr)
1263 buildAssignPtr(
B, ElemTy, Arg);
1267bool SPIRVEmitIntrinsics::runOnFunction(
Function &Func) {
1268 if (
Func.isDeclaration())
1272 GR =
ST.getSPIRVGlobalRegistry();
1277 AggrConstTypes.
clear();
1280 processParamTypesByFunHeader(
F,
B);
1288 Type *ElTy =
SI->getValueOperand()->getType();
1293 B.SetInsertPoint(&
Func.getEntryBlock(),
Func.getEntryBlock().begin());
1294 for (
auto &GV :
Func.getParent()->globals())
1295 processGlobalValue(GV,
B);
1297 preprocessUndefs(
B);
1298 preprocessCompositeConstants(
B);
1303 for (
auto &
I : Worklist) {
1304 insertAssignPtrTypeIntrs(
I,
B);
1305 insertAssignTypeIntrs(
I,
B);
1306 insertPtrCastOrAssignTypeInstr(
I,
B);
1311 deduceOperandElementType(&
I);
1313 for (
auto *
I : Worklist) {
1314 TrackConstants =
true;
1315 if (!
I->getType()->isVoidTy() || isa<StoreInst>(
I))
1316 B.SetInsertPoint(
I->getNextNode());
1322 processInstrAfterVisit(
I,
B);
1328bool SPIRVEmitIntrinsics::runOnModule(
Module &M) {
1329 bool Changed =
false;
1337 if (!
F.isDeclaration() && !
F.isIntrinsic()) {
1339 GR =
ST.getSPIRVGlobalRegistry();
1341 processParamTypes(&
F,
B);
1349 return new SPIRVEmitIntrinsics(
TM);
Expand Atomic instructions
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrConstForceInt32(const Value *V)
static void reportFatalOnTokenType(const Instruction *I)
static Type * getPointeeTypeByCallInst(StringRef DemangledName, Function *CalledF, unsigned OpIdx)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static bool requireAssignType(Instruction *I)
static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB)
static SymbolRef::Type getType(const Symbol *Sym)
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
an instruction to allocate memory on the stack
Represent the analysis usage information of a pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An instruction that atomically checks whether a specified value is in a memory location,...
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
This class represents a no-op cast from one type to another.
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
Implements a dense probed hash-table based set.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
PointerType * getType() const
Global values are always pointers.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Indirect Branch Instruction.
void addDestination(BasicBlock *Dest)
Add a destination.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitInsertValueInst(InsertValueInst &I)
RetTy visitUnreachableInst(UnreachableInst &I)
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
RetTy visitBitCastInst(BitCastInst &I)
RetTy visitSwitchInst(SwitchInst &I)
RetTy visitExtractValueInst(ExtractValueInst &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitAllocaInst(AllocaInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Flags
Flags values. These may be or'd together.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Type * findDeducedCompositeType(const Value *Val)
void addDeducedElementType(Value *Val, Type *Ty)
unsigned getPointerSize() const
void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy)
void addDeducedCompositeType(Value *Val, Type *Ty)
Type * findDeducedElementType(const Value *Val)
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
StringRef getTargetExtName() const
bool isTargetExtTy() const
Return true if this is a target extension type.
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isVoidTy() const
Return true if this is 'void'.
A few GPU targets, such as DXIL and SPIR-V, have typed pointers.
static TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
iterator_range< user_iterator > users()
unsigned getNumUses() const
This method computes the number of uses of this Value.
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ SPIR_KERNEL
Used for SPIR kernel functions.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &)
ModulePass * createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM)
unsigned getPointerAddressSpace(const Type *T)
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)
bool isTypedPointerTy(const Type *T)
bool isPointerTy(const Type *T)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
@ Ref
The access may reference the value stored in memory.
DWARFExpression::Operation Op
Type * getPointeeTypeByAttr(Argument *Arg)
bool hasPointeeTypeAttr(Argument *Arg)
void addStringImm(const StringRef &Str, MCInst &Inst)
bool isUntypedPointerTy(const Type *T)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)