LLVM 20.0.0git
SPIRVEmitIntrinsics.cpp
Go to the documentation of this file.
1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVMetadata.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVTargetMachine.h"
19#include "SPIRVUtils.h"
20#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/InstVisitor.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
25
26#include <queue>
27
28// This pass performs the following transformation on LLVM IR level required
29// for the following translation to SPIR-V:
30// - replaces direct usages of aggregate constants with target-specific
31// intrinsics;
32// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
33// with a target-specific intrinsics;
34// - emits intrinsics for the global variable initializers since IRTranslator
35// doesn't handle them and it's not very convenient to translate them
36// ourselves;
37// - emits intrinsics to keep track of the string names assigned to the values;
38// - emits intrinsics to keep track of constants (this is necessary to have an
39// LLVM IR constant after the IRTranslation is completed) for their further
40// deduplication;
41// - emits intrinsics to keep track of original LLVM types of the values
42// to be able to emit proper SPIR-V types eventually.
43//
44// TODO: consider removing spv.track.constant in favor of spv.assign.type.
45
46using namespace llvm;
47
48namespace llvm {
49namespace SPIRV {
50#define GET_BuiltinGroup_DECL
51#include "SPIRVGenTables.inc"
52} // namespace SPIRV
54} // namespace llvm
55
56namespace {
57
58inline MetadataAsValue *buildMD(Value *Arg) {
59 LLVMContext &Ctx = Arg->getContext();
62}
63
64class SPIRVEmitIntrinsics
65 : public ModulePass,
66 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
67 SPIRVTargetMachine *TM = nullptr;
68 SPIRVGlobalRegistry *GR = nullptr;
69 Function *F = nullptr;
70 bool TrackConstants = true;
73 DenseSet<Instruction *> AggrStores;
74 SPIRV::InstructionSet::InstructionSet InstrSet;
75
76 // a register of Instructions that don't have a complete type definition
77 SmallPtrSet<Value *, 8> UncompleteTypeInfo;
78 SmallVector<Instruction *> PostprocessWorklist;
79
80 // well known result types of builtins
81 enum WellKnownTypes { Event };
82
83 // deduce element type of untyped pointers
84 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
85 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
86 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
87 bool UnknownElemTypeI8);
88 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
89 bool UnknownElemTypeI8);
90 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
91 std::unordered_set<Value *> &Visited,
92 bool UnknownElemTypeI8);
93 Type *deduceElementTypeByUsersDeep(Value *Op,
94 std::unordered_set<Value *> &Visited,
95 bool UnknownElemTypeI8);
96 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
97 bool UnknownElemTypeI8);
98
99 // deduce nested types of composites
100 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
101 Type *deduceNestedTypeHelper(User *U, Type *Ty,
102 std::unordered_set<Value *> &Visited,
103 bool UnknownElemTypeI8);
104
105 // deduce Types of operands of the Instruction if possible
106 void deduceOperandElementType(Instruction *I, Instruction *AskOp = 0,
107 Type *AskTy = 0, CallInst *AssignCI = 0);
108
109 void preprocessCompositeConstants(IRBuilder<> &B);
110 void preprocessUndefs(IRBuilder<> &B);
111
112 CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
113 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
114 IRBuilder<> &B) {
116 Args.push_back(Arg2);
117 Args.push_back(buildMD(Arg));
118 for (auto *Imm : Imms)
119 Args.push_back(Imm);
120 return B.CreateIntrinsic(IntrID, {Types}, Args);
121 }
122
123 void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg);
124 void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg);
125 void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType);
126
127 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
128 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
129 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
130 bool UnknownElemTypeI8);
131 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
132 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
133 IRBuilder<> &B);
134 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
135 Type *ExpectedElementType,
136 unsigned OperandToReplace,
137 IRBuilder<> &B);
138 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
140 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
141 void processParamTypes(Function *F, IRBuilder<> &B);
142 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
143 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
144 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
145 std::unordered_set<Function *> &FVisited);
146
147public:
148 static char ID;
149 SPIRVEmitIntrinsics() : ModulePass(ID) {
151 }
152 SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : ModulePass(ID), TM(_TM) {
154 }
169
170 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
171
172 bool runOnModule(Module &M) override;
173 bool runOnFunction(Function &F);
174 bool postprocessTypes();
175
176 void getAnalysisUsage(AnalysisUsage &AU) const override {
178 }
179};
180
181bool isConvergenceIntrinsic(const Instruction *I) {
182 const auto *II = dyn_cast<IntrinsicInst>(I);
183 if (!II)
184 return false;
185
186 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
187 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
188 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
189}
190} // namespace
191
192char SPIRVEmitIntrinsics::ID = 0;
193
194INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
195 false, false)
196
197static inline bool isAssignTypeInstr(const Instruction *I) {
198 return isa<IntrinsicInst>(I) &&
199 cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
200}
201
203 return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
204 isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
205}
206
207static bool isAggrConstForceInt32(const Value *V) {
208 return isa<ConstantArray>(V) || isa<ConstantStruct>(V) ||
209 isa<ConstantDataArray>(V) ||
210 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
211}
212
214 if (isa<PHINode>(I))
215 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
216 else
217 B.SetInsertPoint(I);
218}
219
221 B.SetCurrentDebugLocation(I->getDebugLoc());
222 if (I->getType()->isVoidTy())
223 B.SetInsertPoint(I->getNextNode());
224 else
225 B.SetInsertPoint(*I->getInsertionPointAfterDef());
226}
227
229 IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
230 if (Intr) {
231 switch (Intr->getIntrinsicID()) {
232 case Intrinsic::invariant_start:
233 case Intrinsic::invariant_end:
234 return false;
235 }
236 }
237 return true;
238}
239
240static inline void reportFatalOnTokenType(const Instruction *I) {
241 if (I->getType()->isTokenTy())
242 report_fatal_error("A token is encountered but SPIR-V without extensions "
243 "does not support token type",
244 false);
245}
246
248 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
249 isPointerTy(SI->getValueOperand()->getType()) &&
250 isa<Argument>(SI->getValueOperand());
251}
252
253// Maybe restore original function return type.
255 Type *Ty) {
256 CallInst *CI = dyn_cast<CallInst>(I);
257 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
259 return Ty;
260 if (Type *OriginalTy = GR->findMutated(CI->getCalledFunction()))
261 return OriginalTy;
262 return Ty;
263}
264
265// Reconstruct type with nested element types according to deduced type info.
266// Return nullptr if no detailed type info is available.
268 Type *Ty = Op->getType();
269 if (!isUntypedPointerTy(Ty))
270 return Ty;
271 // try to find the pointee type
272 if (Type *NestedTy = GR->findDeducedElementType(Op))
274 // not a pointer according to the type info (e.g., Event object)
276 if (!CI)
277 return nullptr;
278 MetadataAsValue *MD = cast<MetadataAsValue>(CI->getArgOperand(1));
279 return cast<ConstantAsMetadata>(MD->getMetadata())->getType();
280}
281
282void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
283 Value *Arg) {
284 Value *OfType = PoisonValue::get(Ty);
285 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
286 {Arg->getType()}, OfType, Arg, {}, B);
287 GR->addAssignPtrTypeInstr(Arg, AssignCI);
288}
289
290void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,
291 Value *Arg) {
292 Value *OfType = PoisonValue::get(ElemTy);
293 CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg);
294 if (AssignPtrTyCI == nullptr ||
295 AssignPtrTyCI->getParent()->getParent() != F) {
296 AssignPtrTyCI = buildIntrWithMD(
297 Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,
298 {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
299 GR->addDeducedElementType(AssignPtrTyCI, ElemTy);
300 GR->addDeducedElementType(Arg, ElemTy);
301 GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI);
302 } else {
303 updateAssignType(AssignPtrTyCI, Arg, OfType);
304 }
305}
306
307void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg,
308 Value *OfType) {
309 AssignCI->setArgOperand(1, buildMD(OfType));
310 if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() !=
311 Intrinsic::spv_assign_ptr_type)
312 return;
313
314 // update association with the pointee type
315 Type *ElemTy = OfType->getType();
316 GR->addDeducedElementType(AssignCI, ElemTy);
317 GR->addDeducedElementType(Arg, ElemTy);
318}
319
320// Set element pointer type to the given value of ValueTy and tries to
321// specify this type further (recursively) by Operand value, if needed.
322Type *
323SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
324 bool UnknownElemTypeI8) {
325 std::unordered_set<Value *> Visited;
326 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
327 UnknownElemTypeI8);
328}
329
330Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
331 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
332 bool UnknownElemTypeI8) {
333 Type *Ty = ValueTy;
334 if (Operand) {
335 if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
336 if (Type *NestedTy =
337 deduceElementTypeHelper(Operand, Visited, UnknownElemTypeI8))
338 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
339 } else {
340 Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited,
341 UnknownElemTypeI8);
342 }
343 }
344 return Ty;
345}
346
347// Traverse User instructions to deduce an element pointer type of the operand.
348Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
349 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
350 if (!Op || !isPointerTy(Op->getType()))
351 return nullptr;
352
353 if (auto ElemTy = getPointeeType(Op->getType()))
354 return ElemTy;
355
356 // maybe we already know operand's element type
357 if (Type *KnownTy = GR->findDeducedElementType(Op))
358 return KnownTy;
359
360 for (User *OpU : Op->users()) {
361 if (Instruction *Inst = dyn_cast<Instruction>(OpU)) {
362 if (Type *Ty = deduceElementTypeHelper(Inst, Visited, UnknownElemTypeI8))
363 return Ty;
364 }
365 }
366 return nullptr;
367}
368
369// Implements what we know in advance about intrinsics and builtin calls
370// TODO: consider feasibility of this particular case to be generalized by
371// encoding knowledge about intrinsics and builtin calls by corresponding
372// specification rules
374 Function *CalledF, unsigned OpIdx) {
375 if ((DemangledName.starts_with("__spirv_ocl_printf(") ||
376 DemangledName.starts_with("printf(")) &&
377 OpIdx == 0)
378 return IntegerType::getInt8Ty(CalledF->getContext());
379 return nullptr;
380}
381
382// Deduce and return a successfully deduced Type of the Instruction,
383// or nullptr otherwise.
384Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
385 bool UnknownElemTypeI8) {
386 std::unordered_set<Value *> Visited;
387 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
388}
389
390void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
391 bool UnknownElemTypeI8) {
392 if (isUntypedPointerTy(RefTy)) {
393 if (!UnknownElemTypeI8)
394 return;
395 if (auto *I = dyn_cast<Instruction>(Op)) {
396 UncompleteTypeInfo.insert(I);
397 PostprocessWorklist.push_back(I);
398 }
399 }
400 Ty = RefTy;
401}
402
403Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
404 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
405 // allow to pass nullptr as an argument
406 if (!I)
407 return nullptr;
408
409 // maybe already known
410 if (Type *KnownTy = GR->findDeducedElementType(I))
411 return KnownTy;
412
413 // maybe a cycle
414 if (Visited.find(I) != Visited.end())
415 return nullptr;
416 Visited.insert(I);
417
418 // fallback value in case when we fail to deduce a type
419 Type *Ty = nullptr;
420 // look for known basic patterns of type inference
421 if (auto *Ref = dyn_cast<AllocaInst>(I)) {
422 maybeAssignPtrType(Ty, I, Ref->getAllocatedType(), UnknownElemTypeI8);
423 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
424 Ty = Ref->getResultElementType();
425 } else if (auto *Ref = dyn_cast<GlobalValue>(I)) {
426 Ty = deduceElementTypeByValueDeep(
427 Ref->getValueType(),
428 Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr, Visited,
429 UnknownElemTypeI8);
430 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
431 Type *RefTy = deduceElementTypeHelper(Ref->getPointerOperand(), Visited,
432 UnknownElemTypeI8);
433 maybeAssignPtrType(Ty, I, RefTy, UnknownElemTypeI8);
434 } else if (auto *Ref = dyn_cast<BitCastInst>(I)) {
435 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
436 isPointerTy(Src) && isPointerTy(Dest))
437 Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited,
438 UnknownElemTypeI8);
439 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
440 Value *Op = Ref->getNewValOperand();
441 if (isPointerTy(Op->getType()))
442 Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
443 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
444 Value *Op = Ref->getValOperand();
445 if (isPointerTy(Op->getType()))
446 Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
447 } else if (auto *Ref = dyn_cast<PHINode>(I)) {
448 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
449 Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited,
450 UnknownElemTypeI8);
451 if (Ty)
452 break;
453 }
454 } else if (auto *Ref = dyn_cast<SelectInst>(I)) {
455 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
456 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
457 if (Ty)
458 break;
459 }
460 } else if (auto *CI = dyn_cast<CallInst>(I)) {
461 static StringMap<unsigned> ResTypeByArg = {
462 {"to_global", 0},
463 {"to_local", 0},
464 {"to_private", 0},
465 {"__spirv_GenericCastToPtr_ToGlobal", 0},
466 {"__spirv_GenericCastToPtr_ToLocal", 0},
467 {"__spirv_GenericCastToPtr_ToPrivate", 0},
468 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
469 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
470 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
471 // TODO: maybe improve performance by caching demangled names
472 if (Function *CalledF = CI->getCalledFunction()) {
473 std::string DemangledName =
474 getOclOrSpirvBuiltinDemangledName(CalledF->getName());
475 if (DemangledName.length() > 0)
476 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName);
477 auto AsArgIt = ResTypeByArg.find(DemangledName);
478 if (AsArgIt != ResTypeByArg.end()) {
479 Ty = deduceElementTypeHelper(CI->getArgOperand(AsArgIt->second),
480 Visited, UnknownElemTypeI8);
481 }
482 }
483 }
484
485 // remember the found relationship
486 if (Ty) {
487 // specify nested types if needed, otherwise return unchanged
488 GR->addDeducedElementType(I, Ty);
489 }
490
491 return Ty;
492}
493
494// Re-create a type of the value if it has untyped pointer fields, also nested.
495// Return the original value type if no corrections of untyped pointer
496// information is found or needed.
497Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
498 bool UnknownElemTypeI8) {
499 std::unordered_set<Value *> Visited;
500 return deduceNestedTypeHelper(U, U->getType(), Visited, UnknownElemTypeI8);
501}
502
503Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
504 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
505 bool UnknownElemTypeI8) {
506 if (!U)
507 return OrigTy;
508
509 // maybe already known
510 if (Type *KnownTy = GR->findDeducedCompositeType(U))
511 return KnownTy;
512
513 // maybe a cycle
514 if (Visited.find(U) != Visited.end())
515 return OrigTy;
516 Visited.insert(U);
517
518 if (dyn_cast<StructType>(OrigTy)) {
520 bool Change = false;
521 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
522 Value *Op = U->getOperand(i);
523 Type *OpTy = Op->getType();
524 Type *Ty = OpTy;
525 if (Op) {
526 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
527 if (Type *NestedTy =
528 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
529 Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
530 } else {
531 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
532 UnknownElemTypeI8);
533 }
534 }
535 Tys.push_back(Ty);
536 Change |= Ty != OpTy;
537 }
538 if (Change) {
539 Type *NewTy = StructType::create(Tys);
540 GR->addDeducedCompositeType(U, NewTy);
541 return NewTy;
542 }
543 } else if (auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) {
544 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) {
545 Type *OpTy = ArrTy->getElementType();
546 Type *Ty = OpTy;
547 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
548 if (Type *NestedTy =
549 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
550 Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
551 } else {
552 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
553 UnknownElemTypeI8);
554 }
555 if (Ty != OpTy) {
556 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
557 GR->addDeducedCompositeType(U, NewTy);
558 return NewTy;
559 }
560 }
561 } else if (auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
562 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) {
563 Type *OpTy = VecTy->getElementType();
564 Type *Ty = OpTy;
565 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
566 if (Type *NestedTy =
567 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
568 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
569 } else {
570 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
571 UnknownElemTypeI8);
572 }
573 if (Ty != OpTy) {
574 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
575 GR->addDeducedCompositeType(U, NewTy);
576 return NewTy;
577 }
578 }
579 }
580
581 return OrigTy;
582}
583
584Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
585 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
586 return Ty;
587 if (!UnknownElemTypeI8)
588 return nullptr;
589 if (auto *Instr = dyn_cast<Instruction>(I)) {
590 UncompleteTypeInfo.insert(Instr);
591 PostprocessWorklist.push_back(Instr);
592 }
593 return IntegerType::getInt8Ty(I->getContext());
594}
595
597 Value *PointerOperand) {
598 Type *PointeeTy = GR->findDeducedElementType(PointerOperand);
599 if (PointeeTy && !isUntypedPointerTy(PointeeTy))
600 return nullptr;
601 auto *PtrTy = dyn_cast<PointerType>(I->getType());
602 if (!PtrTy)
603 return I->getType();
604 if (Type *NestedTy = GR->findDeducedElementType(I))
605 return getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
606 return nullptr;
607}
608
609// If the Instruction has Pointer operands with unresolved types, this function
610// tries to deduce them. If the Instruction has Pointer operands with known
611// types which differ from expected, this function tries to insert a bitcast to
612// resolve the issue.
613void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
614 Instruction *AskOp,
615 Type *AskTy,
616 CallInst *AskCI) {
618 Type *KnownElemTy = nullptr;
619 // look for known basic patterns of type inference
620 if (auto *Ref = dyn_cast<PHINode>(I)) {
621 if (!isPointerTy(I->getType()) ||
622 !(KnownElemTy = GR->findDeducedElementType(I)))
623 return;
624 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
625 Value *Op = Ref->getIncomingValue(i);
626 if (isPointerTy(Op->getType()))
627 Ops.push_back(std::make_pair(Op, i));
628 }
629 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
630 KnownElemTy = GR->findDeducedElementType(I);
631 if (!KnownElemTy)
632 return;
633 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 0));
634 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
635 KnownElemTy = Ref->getSourceElementType();
636 if (isUntypedPointerTy(KnownElemTy))
637 return;
638 Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
639 if (PointeeTy && !isUntypedPointerTy(PointeeTy))
640 return;
641 Ops.push_back(std::make_pair(Ref->getPointerOperand(),
643 } else if (auto *Ref = dyn_cast<LoadInst>(I)) {
644 KnownElemTy = I->getType();
645 if (isUntypedPointerTy(KnownElemTy))
646 return;
647 Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
648 if (PointeeTy && !isUntypedPointerTy(PointeeTy))
649 return;
650 Ops.push_back(std::make_pair(Ref->getPointerOperand(),
652 } else if (auto *Ref = dyn_cast<StoreInst>(I)) {
653 if (IsKernelArgInt8(Ref->getParent()->getParent(), Ref))
654 return;
655 if (!(KnownElemTy = reconstructType(GR, Ref->getValueOperand())))
656 return;
657 Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
658 if (PointeeTy && !isUntypedPointerTy(PointeeTy))
659 return;
660 Ops.push_back(std::make_pair(Ref->getPointerOperand(),
662 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
663 KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
664 if (!KnownElemTy)
665 return;
666 Ops.push_back(std::make_pair(Ref->getPointerOperand(),
668 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
669 KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
670 if (!KnownElemTy)
671 return;
672 Ops.push_back(std::make_pair(Ref->getPointerOperand(),
674 } else if (auto *Ref = dyn_cast<SelectInst>(I)) {
675 if (!isPointerTy(I->getType()) ||
676 !(KnownElemTy = GR->findDeducedElementType(I)))
677 return;
678 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
679 Value *Op = Ref->getOperand(i);
680 if (isPointerTy(Op->getType()))
681 Ops.push_back(std::make_pair(Op, i));
682 }
683 } else if (auto *Ref = dyn_cast<ReturnInst>(I)) {
684 Type *RetTy = F->getReturnType();
685 if (!isPointerTy(RetTy))
686 return;
687 Value *Op = Ref->getReturnValue();
688 if (!Op)
689 return;
690 if (!(KnownElemTy = GR->findDeducedElementType(F))) {
691 if (Type *OpElemTy = GR->findDeducedElementType(Op)) {
692 GR->addDeducedElementType(F, OpElemTy);
693 TypedPointerType *DerivedTy =
695 GR->addReturnType(F, DerivedTy);
696 }
697 return;
698 }
699 Ops.push_back(std::make_pair(Op, 0));
700 } else if (auto *Ref = dyn_cast<ICmpInst>(I)) {
701 if (!isPointerTy(Ref->getOperand(0)->getType()))
702 return;
703 Value *Op0 = Ref->getOperand(0);
704 Value *Op1 = Ref->getOperand(1);
705 Type *ElemTy0 = GR->findDeducedElementType(Op0);
706 Type *ElemTy1 = GR->findDeducedElementType(Op1);
707 if (ElemTy0) {
708 KnownElemTy = ElemTy0;
709 Ops.push_back(std::make_pair(Op1, 1));
710 } else if (ElemTy1) {
711 KnownElemTy = ElemTy1;
712 Ops.push_back(std::make_pair(Op0, 0));
713 }
714 } else if (auto *CI = dyn_cast<CallInst>(I)) {
715 if (Function *CalledF = CI->getCalledFunction()) {
716 std::string DemangledName =
717 getOclOrSpirvBuiltinDemangledName(CalledF->getName());
718 if (DemangledName.length() > 0 &&
719 !StringRef(DemangledName).starts_with("llvm.")) {
720 auto [Grp, Opcode, ExtNo] =
721 SPIRV::mapBuiltinToOpcode(DemangledName, InstrSet);
722 if (Opcode == SPIRV::OpGroupAsyncCopy) {
723 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2;
724 ++i) {
725 Value *Op = CI->getArgOperand(i);
726 if (!isPointerTy(Op->getType()))
727 continue;
728 ++PtrCnt;
729 if (Type *ElemTy = GR->findDeducedElementType(Op))
730 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
731 Ops.push_back(std::make_pair(Op, i));
732 }
733 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
734 if (CI->arg_size() < 2)
735 return;
736 Value *Op = CI->getArgOperand(0);
737 if (!isPointerTy(Op->getType()))
738 return;
739 switch (Opcode) {
740 case SPIRV::OpAtomicLoad:
741 case SPIRV::OpAtomicCompareExchangeWeak:
742 case SPIRV::OpAtomicCompareExchange:
743 case SPIRV::OpAtomicExchange:
744 case SPIRV::OpAtomicIAdd:
745 case SPIRV::OpAtomicISub:
746 case SPIRV::OpAtomicOr:
747 case SPIRV::OpAtomicXor:
748 case SPIRV::OpAtomicAnd:
749 case SPIRV::OpAtomicUMin:
750 case SPIRV::OpAtomicUMax:
751 case SPIRV::OpAtomicSMin:
752 case SPIRV::OpAtomicSMax: {
753 KnownElemTy = getAtomicElemTy(GR, I, Op);
754 if (!KnownElemTy)
755 return;
756 Ops.push_back(std::make_pair(Op, 0));
757 } break;
758 }
759 }
760 }
761 }
762 }
763
764 // There is no enough info to deduce types or all is valid.
765 if (!KnownElemTy || Ops.size() == 0)
766 return;
767
768 LLVMContext &Ctx = F->getContext();
769 IRBuilder<> B(Ctx);
770 for (auto &OpIt : Ops) {
771 Value *Op = OpIt.first;
772 if (Op->use_empty() || (AskOp && Op != AskOp))
773 continue;
774 Type *Ty = AskOp ? AskTy : GR->findDeducedElementType(Op);
775 if (Ty == KnownElemTy)
776 continue;
777 Value *OpTyVal = PoisonValue::get(KnownElemTy);
778 Type *OpTy = Op->getType();
779 if (!Ty || AskTy || isUntypedPointerTy(Ty) ||
780 UncompleteTypeInfo.contains(Op)) {
781 GR->addDeducedElementType(Op, KnownElemTy);
782 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
783 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Op);
784 if (AssignCI == nullptr) {
785 Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get());
786 setInsertPointSkippingPhis(B, User ? User->getNextNode() : I);
787 CallInst *CI =
788 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op,
789 {B.getInt32(getPointerAddressSpace(OpTy))}, B);
790 GR->addAssignPtrTypeInstr(Op, CI);
791 } else {
792 updateAssignType(AssignCI, Op, OpTyVal);
793 }
794 } else {
795 if (auto *OpI = dyn_cast<Instruction>(Op)) {
796 // spv_ptrcast's argument Op denotes an instruction that generates
797 // a value, and we may use getInsertionPointAfterDef()
798 B.SetInsertPoint(*OpI->getInsertionPointAfterDef());
799 B.SetCurrentDebugLocation(OpI->getDebugLoc());
800 } else if (auto *OpA = dyn_cast<Argument>(Op)) {
801 B.SetInsertPointPastAllocas(OpA->getParent());
802 B.SetCurrentDebugLocation(DebugLoc());
803 } else {
804 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
805 }
806 SmallVector<Type *, 2> Types = {OpTy, OpTy};
807 SmallVector<Value *, 2> Args = {Op, buildMD(OpTyVal),
808 B.getInt32(getPointerAddressSpace(OpTy))};
809 CallInst *PtrCastI =
810 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
811 I->setOperand(OpIt.second, PtrCastI);
812 }
813 }
814}
815
816void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
817 Instruction *New,
818 IRBuilder<> &B) {
819 while (!Old->user_empty()) {
820 auto *U = Old->user_back();
821 if (isAssignTypeInstr(U)) {
822 B.SetInsertPoint(U);
823 SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
824 CallInst *AssignCI =
825 B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
826 GR->addAssignPtrTypeInstr(New, AssignCI);
827 U->eraseFromParent();
828 } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
829 isa<CallInst>(U)) {
830 U->replaceUsesOfWith(Old, New);
831 } else {
832 llvm_unreachable("illegal aggregate intrinsic user");
833 }
834 }
835 Old->eraseFromParent();
836}
837
838void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
839 std::queue<Instruction *> Worklist;
840 for (auto &I : instructions(F))
841 Worklist.push(&I);
842
843 while (!Worklist.empty()) {
844 Instruction *I = Worklist.front();
845 bool BPrepared = false;
846 Worklist.pop();
847
848 for (auto &Op : I->operands()) {
849 auto *AggrUndef = dyn_cast<UndefValue>(Op);
850 if (!AggrUndef || !Op->getType()->isAggregateType())
851 continue;
852
853 if (!BPrepared) {
855 BPrepared = true;
856 }
857 auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {}, {});
858 Worklist.push(IntrUndef);
859 I->replaceUsesOfWith(Op, IntrUndef);
860 AggrConsts[IntrUndef] = AggrUndef;
861 AggrConstTypes[IntrUndef] = AggrUndef->getType();
862 }
863 }
864}
865
866void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
867 std::queue<Instruction *> Worklist;
868 for (auto &I : instructions(F))
869 Worklist.push(&I);
870
871 while (!Worklist.empty()) {
872 auto *I = Worklist.front();
873 bool IsPhi = isa<PHINode>(I), BPrepared = false;
874 assert(I);
875 bool KeepInst = false;
876 for (const auto &Op : I->operands()) {
877 Constant *AggrConst = nullptr;
878 Type *ResTy = nullptr;
879 if (auto *COp = dyn_cast<ConstantVector>(Op)) {
880 AggrConst = cast<Constant>(COp);
881 ResTy = COp->getType();
882 } else if (auto *COp = dyn_cast<ConstantArray>(Op)) {
883 AggrConst = cast<Constant>(COp);
884 ResTy = B.getInt32Ty();
885 } else if (auto *COp = dyn_cast<ConstantStruct>(Op)) {
886 AggrConst = cast<Constant>(COp);
887 ResTy = B.getInt32Ty();
888 } else if (auto *COp = dyn_cast<ConstantDataArray>(Op)) {
889 AggrConst = cast<Constant>(COp);
890 ResTy = B.getInt32Ty();
891 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Op)) {
892 AggrConst = cast<Constant>(COp);
893 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
894 }
895 if (AggrConst) {
897 if (auto *COp = dyn_cast<ConstantDataSequential>(Op))
898 for (unsigned i = 0; i < COp->getNumElements(); ++i)
899 Args.push_back(COp->getElementAsConstant(i));
900 else
901 for (auto &COp : AggrConst->operands())
902 Args.push_back(COp);
903 if (!BPrepared) {
904 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
905 : B.SetInsertPoint(I);
906 BPrepared = true;
907 }
908 auto *CI =
909 B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args});
910 Worklist.push(CI);
911 I->replaceUsesOfWith(Op, CI);
912 KeepInst = true;
913 AggrConsts[CI] = AggrConst;
914 AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst, false);
915 }
916 }
917 if (!KeepInst)
918 Worklist.pop();
919 }
920}
921
922Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
923 if (!Call.isInlineAsm())
924 return &Call;
925
926 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
927 LLVMContext &Ctx = F->getContext();
928
929 Constant *TyC = UndefValue::get(IA->getFunctionType());
930 MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
932 buildMD(TyC),
933 MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))};
934 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
935 Args.push_back(Call.getArgOperand(OpIdx));
936
937 IRBuilder<> B(Call.getParent());
938 B.SetInsertPoint(&Call);
939 B.CreateIntrinsic(Intrinsic::spv_inline_asm, {}, {Args});
940 return &Call;
941}
942
943Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
944 BasicBlock *ParentBB = I.getParent();
945 IRBuilder<> B(ParentBB);
946 B.SetInsertPoint(&I);
949 for (auto &Op : I.operands()) {
950 if (Op.get()->getType()->isSized()) {
951 Args.push_back(Op);
952 } else if (BasicBlock *BB = dyn_cast<BasicBlock>(Op.get())) {
953 BBCases.push_back(BB);
954 Args.push_back(BlockAddress::get(BB->getParent(), BB));
955 } else {
956 report_fatal_error("Unexpected switch operand");
957 }
958 }
959 CallInst *NewI = B.CreateIntrinsic(Intrinsic::spv_switch,
960 {I.getOperand(0)->getType()}, {Args});
961 // remove switch to avoid its unneeded and undesirable unwrap into branches
962 // and conditions
963 I.replaceAllUsesWith(NewI);
964 I.eraseFromParent();
965 // insert artificial and temporary instruction to preserve valid CFG,
966 // it will be removed after IR translation pass
967 B.SetInsertPoint(ParentBB);
968 IndirectBrInst *BrI = B.CreateIndirectBr(
969 Constant::getNullValue(PointerType::getUnqual(ParentBB->getContext())),
970 BBCases.size());
971 for (BasicBlock *BBCase : BBCases)
972 BrI->addDestination(BBCase);
973 return BrI;
974}
975
976Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
977 IRBuilder<> B(I.getParent());
978 B.SetInsertPoint(&I);
979 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
981 Args.push_back(B.getInt1(I.isInBounds()));
982 for (auto &Op : I.operands())
983 Args.push_back(Op);
984 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
985 I.replaceAllUsesWith(NewI);
986 I.eraseFromParent();
987 return NewI;
988}
989
990Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
991 IRBuilder<> B(I.getParent());
992 B.SetInsertPoint(&I);
993 Value *Source = I.getOperand(0);
994
995 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
996 // varying element types. In case of IR coming from older versions of LLVM
997 // such bitcasts do not provide sufficient information, should be just skipped
998 // here, and handled in insertPtrCastOrAssignTypeInstr.
999 if (isPointerTy(I.getType())) {
1000 I.replaceAllUsesWith(Source);
1001 I.eraseFromParent();
1002 return nullptr;
1003 }
1004
1005 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1006 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1007 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
1008 std::string InstName = I.hasName() ? I.getName().str() : "";
1009 I.replaceAllUsesWith(NewI);
1010 I.eraseFromParent();
1011 NewI->setName(InstName);
1012 return NewI;
1013}
1014
1015void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1016 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1017 Type *VTy = V->getType();
1018
1019 // A couple of sanity checks.
1020 assert(isPointerTy(VTy) && "Expect a pointer type!");
1021 if (auto PType = dyn_cast<TypedPointerType>(VTy))
1022 if (PType->getElementType() != AssignedType)
1023 report_fatal_error("Unexpected pointer element type!");
1024
1025 CallInst *AssignCI = GR->findAssignPtrTypeInstr(V);
1026 if (!AssignCI) {
1027 buildAssignType(B, AssignedType, V);
1028 return;
1029 }
1030
1031 Type *CurrentType =
1032 dyn_cast<ConstantAsMetadata>(
1033 cast<MetadataAsValue>(AssignCI->getOperand(1))->getMetadata())
1034 ->getType();
1035 if (CurrentType == AssignedType)
1036 return;
1037
1038 // Builtin types cannot be redeclared or casted.
1039 if (CurrentType->isTargetExtTy())
1040 report_fatal_error("Type mismatch " + CurrentType->getTargetExtName() +
1041 "/" + AssignedType->getTargetExtName() +
1042 " for value " + V->getName(),
1043 false);
1044
1045 // Our previous guess about the type seems to be wrong, let's update
1046 // inferred type according to a new, more precise type information.
1047 updateAssignType(AssignCI, V, PoisonValue::get(AssignedType));
1048}
1049
1050void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1051 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1052 unsigned OperandToReplace, IRBuilder<> &B) {
1053 // If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source
1054 // pointer instead. The BitCastInst should be later removed when visited.
1055 while (BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
1056 Pointer = BC->getOperand(0);
1057
1058 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1059 Type *PointerElemTy = deduceElementTypeHelper(Pointer, false);
1060 if (PointerElemTy == ExpectedElementType ||
1061 isEquivalentTypes(PointerElemTy, ExpectedElementType))
1062 return;
1063
1065 MetadataAsValue *VMD = buildMD(PoisonValue::get(ExpectedElementType));
1066 unsigned AddressSpace = getPointerAddressSpace(Pointer->getType());
1067 bool FirstPtrCastOrAssignPtrType = true;
1068
1069 // Do not emit new spv_ptrcast if equivalent one already exists or when
1070 // spv_assign_ptr_type already targets this pointer with the same element
1071 // type.
1072 for (auto User : Pointer->users()) {
1073 auto *II = dyn_cast<IntrinsicInst>(User);
1074 if (!II ||
1075 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1076 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1077 II->getOperand(0) != Pointer)
1078 continue;
1079
1080 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1081 // pointer.
1082 FirstPtrCastOrAssignPtrType = false;
1083 if (II->getOperand(1) != VMD ||
1084 dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() !=
1086 continue;
1087
1088 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the same
1089 // element type and address space.
1090 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1091 return;
1092
1093 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1094 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1095 if (II->getParent() != I->getParent())
1096 continue;
1097
1098 I->setOperand(OperandToReplace, II);
1099 return;
1100 }
1101
1102 // // Do not emit spv_ptrcast if it would cast to the default pointer element
1103 // // type (i8) of the same address space.
1104 // if (ExpectedElementType->isIntegerTy(8))
1105 // return;
1106
1107 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and emit
1108 // spv_assign_ptr_type instead.
1109 if (FirstPtrCastOrAssignPtrType &&
1110 (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
1111 buildAssignPtr(B, ExpectedElementType, Pointer);
1112 return;
1113 }
1114
1115 // Emit spv_ptrcast
1116 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1118 auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
1119 I->setOperand(OperandToReplace, PtrCastI);
1120}
1121
1122void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1123 IRBuilder<> &B) {
1124 // Handle basic instructions:
1125 StoreInst *SI = dyn_cast<StoreInst>(I);
1126 if (IsKernelArgInt8(F, SI)) {
1127 return replacePointerOperandWithPtrCast(
1128 I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0,
1129 B);
1130 } else if (SI) {
1131 Value *Op = SI->getValueOperand();
1132 Type *OpTy = Op->getType();
1133 if (auto *OpI = dyn_cast<Instruction>(Op))
1134 OpTy = restoreMutatedType(GR, OpI, OpTy);
1135 if (OpTy == Op->getType())
1136 OpTy = deduceElementTypeByValueDeep(OpTy, Op, false);
1137 return replacePointerOperandWithPtrCast(I, SI->getPointerOperand(), OpTy, 1,
1138 B);
1139 } else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
1140 return replacePointerOperandWithPtrCast(I, LI->getPointerOperand(),
1141 LI->getType(), 0, B);
1142 } else if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
1143 return replacePointerOperandWithPtrCast(I, GEPI->getPointerOperand(),
1144 GEPI->getSourceElementType(), 0, B);
1145 }
1146
1147 // Handle calls to builtins (non-intrinsics):
1148 CallInst *CI = dyn_cast<CallInst>(I);
1149 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
1151 return;
1152
1153 // collect information about formal parameter types
1154 std::string DemangledName =
1156 Function *CalledF = CI->getCalledFunction();
1157 SmallVector<Type *, 4> CalledArgTys;
1158 bool HaveTypes = false;
1159 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
1160 Argument *CalledArg = CalledF->getArg(OpIdx);
1161 Type *ArgType = CalledArg->getType();
1162 if (!isPointerTy(ArgType)) {
1163 CalledArgTys.push_back(nullptr);
1164 } else if (isTypedPointerTy(ArgType)) {
1165 CalledArgTys.push_back(cast<TypedPointerType>(ArgType)->getElementType());
1166 HaveTypes = true;
1167 } else {
1168 Type *ElemTy = GR->findDeducedElementType(CalledArg);
1169 if (!ElemTy && hasPointeeTypeAttr(CalledArg))
1170 ElemTy = getPointeeTypeByAttr(CalledArg);
1171 if (!ElemTy) {
1172 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
1173 if (ElemTy) {
1174 GR->addDeducedElementType(CalledArg, ElemTy);
1175 } else {
1176 for (User *U : CalledArg->users()) {
1177 if (Instruction *Inst = dyn_cast<Instruction>(U)) {
1178 if ((ElemTy = deduceElementTypeHelper(Inst, false)) != nullptr)
1179 break;
1180 }
1181 }
1182 }
1183 }
1184 HaveTypes |= ElemTy != nullptr;
1185 CalledArgTys.push_back(ElemTy);
1186 }
1187 }
1188
1189 if (DemangledName.empty() && !HaveTypes)
1190 return;
1191
1192 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
1193 Value *ArgOperand = CI->getArgOperand(OpIdx);
1194 if (!isPointerTy(ArgOperand->getType()))
1195 continue;
1196
1197 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
1198 if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) {
1199 // However, we may have assumptions about the formal argument's type and
1200 // may have a need to insert a ptr cast for the actual parameter of this
1201 // call.
1202 Argument *CalledArg = CalledF->getArg(OpIdx);
1203 if (!GR->findDeducedElementType(CalledArg))
1204 continue;
1205 }
1206
1207 Type *ExpectedType =
1208 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
1209 if (!ExpectedType && !DemangledName.empty())
1211 DemangledName, OpIdx, I->getContext());
1212 if (!ExpectedType || ExpectedType->isVoidTy())
1213 continue;
1214
1215 if (ExpectedType->isTargetExtTy())
1216 insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType),
1217 ArgOperand, B);
1218 else
1219 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B);
1220 }
1221}
1222
1223Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
1224 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
1225 I.getOperand(1)->getType(),
1226 I.getOperand(2)->getType()};
1227 IRBuilder<> B(I.getParent());
1228 B.SetInsertPoint(&I);
1229 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1230 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
1231 std::string InstName = I.hasName() ? I.getName().str() : "";
1232 I.replaceAllUsesWith(NewI);
1233 I.eraseFromParent();
1234 NewI->setName(InstName);
1235 return NewI;
1236}
1237
1239SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
1240 IRBuilder<> B(I.getParent());
1241 B.SetInsertPoint(&I);
1242 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
1243 I.getIndexOperand()->getType()};
1244 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
1245 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
1246 std::string InstName = I.hasName() ? I.getName().str() : "";
1247 I.replaceAllUsesWith(NewI);
1248 I.eraseFromParent();
1249 NewI->setName(InstName);
1250 return NewI;
1251}
1252
1253Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
1254 IRBuilder<> B(I.getParent());
1255 B.SetInsertPoint(&I);
1256 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
1258 for (auto &Op : I.operands())
1259 if (isa<UndefValue>(Op))
1260 Args.push_back(UndefValue::get(B.getInt32Ty()));
1261 else
1262 Args.push_back(Op);
1263 for (auto &Op : I.indices())
1264 Args.push_back(B.getInt32(Op));
1265 Instruction *NewI =
1266 B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
1267 replaceMemInstrUses(&I, NewI, B);
1268 return NewI;
1269}
1270
1271Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
1272 IRBuilder<> B(I.getParent());
1273 B.SetInsertPoint(&I);
1275 for (auto &Op : I.operands())
1276 Args.push_back(Op);
1277 for (auto &Op : I.indices())
1278 Args.push_back(B.getInt32(Op));
1279 auto *NewI =
1280 B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
1281 I.replaceAllUsesWith(NewI);
1282 I.eraseFromParent();
1283 return NewI;
1284}
1285
1286Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
1287 if (!I.getType()->isAggregateType())
1288 return &I;
1289 IRBuilder<> B(I.getParent());
1290 B.SetInsertPoint(&I);
1291 TrackConstants = false;
1292 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
1294 TLI->getLoadMemOperandFlags(I, F->getDataLayout());
1295 auto *NewI =
1296 B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
1297 {I.getPointerOperand(), B.getInt16(Flags),
1298 B.getInt8(I.getAlign().value())});
1299 replaceMemInstrUses(&I, NewI, B);
1300 return NewI;
1301}
1302
1303Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
1304 if (!AggrStores.contains(&I))
1305 return &I;
1306 IRBuilder<> B(I.getParent());
1307 B.SetInsertPoint(&I);
1308 TrackConstants = false;
1309 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
1311 TLI->getStoreMemOperandFlags(I, F->getDataLayout());
1312 auto *PtrOp = I.getPointerOperand();
1313 auto *NewI = B.CreateIntrinsic(
1314 Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
1315 {I.getValueOperand(), PtrOp, B.getInt16(Flags),
1316 B.getInt8(I.getAlign().value())});
1317 I.eraseFromParent();
1318 return NewI;
1319}
1320
1321Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
1322 Value *ArraySize = nullptr;
1323 if (I.isArrayAllocation()) {
1324 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
1325 if (!STI->canUseExtension(
1326 SPIRV::Extension::SPV_INTEL_variable_length_array))
1328 "array allocation: this instruction requires the following "
1329 "SPIR-V extension: SPV_INTEL_variable_length_array",
1330 false);
1331 ArraySize = I.getArraySize();
1332 }
1333 IRBuilder<> B(I.getParent());
1334 B.SetInsertPoint(&I);
1335 TrackConstants = false;
1336 Type *PtrTy = I.getType();
1337 auto *NewI =
1338 ArraySize ? B.CreateIntrinsic(Intrinsic::spv_alloca_array,
1339 {PtrTy, ArraySize->getType()}, {ArraySize})
1340 : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
1341 std::string InstName = I.hasName() ? I.getName().str() : "";
1342 I.replaceAllUsesWith(NewI);
1343 I.eraseFromParent();
1344 NewI->setName(InstName);
1345 return NewI;
1346}
1347
1348Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1349 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
1350 IRBuilder<> B(I.getParent());
1351 B.SetInsertPoint(&I);
1353 for (auto &Op : I.operands())
1354 Args.push_back(Op);
1355 Args.push_back(B.getInt32(I.getSyncScopeID()));
1356 Args.push_back(B.getInt32(
1357 static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering()))));
1358 Args.push_back(B.getInt32(
1359 static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering()))));
1360 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_cmpxchg,
1361 {I.getPointerOperand()->getType()}, {Args});
1362 replaceMemInstrUses(&I, NewI, B);
1363 return NewI;
1364}
1365
1366Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
1367 IRBuilder<> B(I.getParent());
1368 B.SetInsertPoint(&I);
1369 B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
1370 return &I;
1371}
1372
1373void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
1374 IRBuilder<> &B) {
1375 // Skip special artifical variable llvm.global.annotations.
1376 if (GV.getName() == "llvm.global.annotations")
1377 return;
1378 if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
1379 // Deduce element type and store results in Global Registry.
1380 // Result is ignored, because TypedPointerType is not supported
1381 // by llvm IR general logic.
1382 deduceElementTypeHelper(&GV, false);
1384 Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType();
1385 Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init;
1386 auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global,
1387 {GV.getType(), Ty}, {&GV, Const});
1388 InitInst->setArgOperand(1, Init);
1389 }
1390 if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
1391 GV.getNumUses() == 0)
1392 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
1393}
1394
1395// Return true, if we can't decide what is the pointee type now and will get
1396// back to the question later. Return false is spv_assign_ptr_type is not needed
1397// or can be inserted immediately.
1398bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
1399 IRBuilder<> &B,
1400 bool UnknownElemTypeI8) {
1402 if (!isPointerTy(I->getType()) || !requireAssignType(I) ||
1403 isa<BitCastInst>(I))
1404 return false;
1405
1407 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
1408 buildAssignPtr(B, ElemTy, I);
1409 return false;
1410 }
1411 return true;
1412}
1413
1414void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
1415 IRBuilder<> &B) {
1416 // TODO: extend the list of functions with known result types
1417 static StringMap<unsigned> ResTypeWellKnown = {
1418 {"async_work_group_copy", WellKnownTypes::Event},
1419 {"async_work_group_strided_copy", WellKnownTypes::Event},
1420 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
1421
1423
1424 bool IsKnown = false;
1425 if (auto *CI = dyn_cast<CallInst>(I)) {
1426 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
1427 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
1428 Function *CalledF = CI->getCalledFunction();
1429 std::string DemangledName =
1431 if (DemangledName.length() > 0)
1432 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName);
1433 auto ResIt = ResTypeWellKnown.find(DemangledName);
1434 if (ResIt != ResTypeWellKnown.end()) {
1435 IsKnown = true;
1437 switch (ResIt->second) {
1438 case WellKnownTypes::Event:
1439 buildAssignType(B, TargetExtType::get(I->getContext(), "spirv.Event"),
1440 I);
1441 break;
1442 }
1443 }
1444 }
1445 }
1446
1447 Type *Ty = I->getType();
1448 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(Ty) && requireAssignType(I)) {
1450 Type *TypeToAssign = Ty;
1451 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1452 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
1453 II->getIntrinsicID() == Intrinsic::spv_undef) {
1454 auto It = AggrConstTypes.find(II);
1455 if (It == AggrConstTypes.end())
1456 report_fatal_error("Unknown composite intrinsic type");
1457 TypeToAssign = It->second;
1458 }
1459 }
1460 TypeToAssign = restoreMutatedType(GR, I, TypeToAssign);
1461 buildAssignType(B, TypeToAssign, I);
1462 }
1463 for (const auto &Op : I->operands()) {
1464 if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
1465 // Check GetElementPtrConstantExpr case.
1466 (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
1468 Type *OpTy = Op->getType();
1469 if (isa<UndefValue>(Op) && OpTy->isAggregateType()) {
1470 CallInst *AssignCI =
1471 buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op,
1472 UndefValue::get(B.getInt32Ty()), {}, B);
1473 GR->addAssignPtrTypeInstr(Op, AssignCI);
1474 } else if (!isa<Instruction>(Op)) {
1475 Type *OpTy = Op->getType();
1476 if (auto PType = dyn_cast<TypedPointerType>(OpTy)) {
1477 buildAssignPtr(B, PType->getElementType(), Op);
1478 } else if (isPointerTy(OpTy)) {
1479 Type *ElemTy = GR->findDeducedElementType(Op);
1480 buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op);
1481 } else {
1482 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
1483 {OpTy}, Op, Op, {}, B);
1484 GR->addAssignPtrTypeInstr(Op, AssignCI);
1485 }
1486 }
1487 }
1488 }
1489}
1490
1491void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
1492 IRBuilder<> &B) {
1493 if (MDNode *MD = I->getMetadata("spirv.Decorations")) {
1495 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
1496 {I, MetadataAsValue::get(I->getContext(), MD)});
1497 }
1498}
1499
1500void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
1501 IRBuilder<> &B) {
1502 auto *II = dyn_cast<IntrinsicInst>(I);
1503 if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
1504 TrackConstants) {
1506 auto t = AggrConsts.find(I);
1507 assert(t != AggrConsts.end());
1508 auto *NewOp =
1509 buildIntrWithMD(Intrinsic::spv_track_constant,
1510 {II->getType(), II->getType()}, t->second, I, {}, B);
1511 I->replaceAllUsesWith(NewOp);
1512 NewOp->setArgOperand(0, I);
1513 }
1514 bool IsPhi = isa<PHINode>(I), BPrepared = false;
1515 for (const auto &Op : I->operands()) {
1516 if (isa<PHINode>(I) || isa<SwitchInst>(I))
1517 TrackConstants = false;
1518 if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) {
1519 unsigned OpNo = Op.getOperandNo();
1520 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
1521 (II->paramHasAttr(OpNo, Attribute::ImmArg))))
1522 continue;
1523 if (!BPrepared) {
1524 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1525 : B.SetInsertPoint(I);
1526 BPrepared = true;
1527 }
1528 Value *OpTyVal = Op;
1529 if (Op->getType()->isTargetExtTy())
1530 OpTyVal = PoisonValue::get(Op->getType());
1531 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
1532 {Op->getType(), OpTyVal->getType()}, Op,
1533 OpTyVal, {}, B);
1534 I->setOperand(OpNo, NewOp);
1535 }
1536 }
1537 if (I->hasName()) {
1540 std::vector<Value *> Args = {I};
1541 addStringImm(I->getName(), B, Args);
1542 B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
1543 }
1544}
1545
1546Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
1547 unsigned OpIdx) {
1548 std::unordered_set<Function *> FVisited;
1549 return deduceFunParamElementType(F, OpIdx, FVisited);
1550}
1551
1552Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
1553 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
1554 // maybe a cycle
1555 if (FVisited.find(F) != FVisited.end())
1556 return nullptr;
1557 FVisited.insert(F);
1558
1559 std::unordered_set<Value *> Visited;
1561 // search in function's call sites
1562 for (User *U : F->users()) {
1563 CallInst *CI = dyn_cast<CallInst>(U);
1564 if (!CI || OpIdx >= CI->arg_size())
1565 continue;
1566 Value *OpArg = CI->getArgOperand(OpIdx);
1567 if (!isPointerTy(OpArg->getType()))
1568 continue;
1569 // maybe we already know operand's element type
1570 if (Type *KnownTy = GR->findDeducedElementType(OpArg))
1571 return KnownTy;
1572 // try to deduce from the operand itself
1573 Visited.clear();
1574 if (Type *Ty = deduceElementTypeHelper(OpArg, Visited, false))
1575 return Ty;
1576 // search in actual parameter's users
1577 for (User *OpU : OpArg->users()) {
1578 Instruction *Inst = dyn_cast<Instruction>(OpU);
1579 if (!Inst || Inst == CI)
1580 continue;
1581 Visited.clear();
1582 if (Type *Ty = deduceElementTypeHelper(Inst, Visited, false))
1583 return Ty;
1584 }
1585 // check if it's a formal parameter of the outer function
1586 if (!CI->getParent() || !CI->getParent()->getParent())
1587 continue;
1588 Function *OuterF = CI->getParent()->getParent();
1589 if (FVisited.find(OuterF) != FVisited.end())
1590 continue;
1591 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
1592 if (OuterF->getArg(i) == OpArg) {
1593 Lookup.push_back(std::make_pair(OuterF, i));
1594 break;
1595 }
1596 }
1597 }
1598
1599 // search in function parameters
1600 for (auto &Pair : Lookup) {
1601 if (Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
1602 return Ty;
1603 }
1604
1605 return nullptr;
1606}
1607
1608void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
1609 IRBuilder<> &B) {
1610 B.SetInsertPointPastAllocas(F);
1611 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
1612 Argument *Arg = F->getArg(OpIdx);
1613 if (!isUntypedPointerTy(Arg->getType()))
1614 continue;
1615 Type *ElemTy = GR->findDeducedElementType(Arg);
1616 if (!ElemTy && hasPointeeTypeAttr(Arg) &&
1617 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr)
1618 buildAssignPtr(B, ElemTy, Arg);
1619 }
1620}
1621
1622void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
1623 B.SetInsertPointPastAllocas(F);
1624 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
1625 Argument *Arg = F->getArg(OpIdx);
1626 if (!isUntypedPointerTy(Arg->getType()))
1627 continue;
1628 Type *ElemTy = GR->findDeducedElementType(Arg);
1629 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr)
1630 buildAssignPtr(B, ElemTy, Arg);
1631 }
1632}
1633
1634bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
1635 if (Func.isDeclaration())
1636 return false;
1637
1638 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(Func);
1639 GR = ST.getSPIRVGlobalRegistry();
1640 InstrSet = ST.isOpenCLEnv() ? SPIRV::InstructionSet::OpenCL_std
1641 : SPIRV::InstructionSet::GLSL_std_450;
1642
1643 F = &Func;
1644 IRBuilder<> B(Func.getContext());
1645 AggrConsts.clear();
1646 AggrConstTypes.clear();
1647 AggrStores.clear();
1648
1649 processParamTypesByFunHeader(F, B);
1650
1651 // StoreInst's operand type can be changed during the next transformations,
1652 // so we need to store it in the set. Also store already transformed types.
1653 for (auto &I : instructions(Func)) {
1654 StoreInst *SI = dyn_cast<StoreInst>(&I);
1655 if (!SI)
1656 continue;
1657 Type *ElTy = SI->getValueOperand()->getType();
1658 if (ElTy->isAggregateType() || ElTy->isVectorTy())
1659 AggrStores.insert(&I);
1660 }
1661
1662 B.SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin());
1663 for (auto &GV : Func.getParent()->globals())
1664 processGlobalValue(GV, B);
1665
1666 preprocessUndefs(B);
1667 preprocessCompositeConstants(B);
1669 for (auto &I : instructions(Func))
1670 Worklist.push_back(&I);
1671
1672 for (auto &I : Worklist) {
1673 // Don't emit intrinsincs for convergence intrinsics.
1674 if (isConvergenceIntrinsic(I))
1675 continue;
1676
1677 bool Postpone = insertAssignPtrTypeIntrs(I, B, false);
1678 // if Postpone is true, we can't decide on pointee type yet
1679 insertAssignTypeIntrs(I, B);
1680 insertPtrCastOrAssignTypeInstr(I, B);
1682 // if instruction requires a pointee type set, let's check if we know it
1683 // already, and force it to be i8 if not
1684 if (Postpone && !GR->findAssignPtrTypeInstr(I))
1685 insertAssignPtrTypeIntrs(I, B, true);
1686 }
1687
1688 for (auto &I : instructions(Func))
1689 deduceOperandElementType(&I);
1690
1691 for (auto *I : Worklist) {
1692 TrackConstants = true;
1693 if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
1695 // Visitors return either the original/newly created instruction for further
1696 // processing, nullptr otherwise.
1697 I = visit(*I);
1698 if (!I)
1699 continue;
1700
1701 // Don't emit intrinsics for convergence operations.
1702 if (isConvergenceIntrinsic(I))
1703 continue;
1704
1705 processInstrAfterVisit(I, B);
1706 }
1707
1708 return true;
1709}
1710
1711// Try to deduce a better type for pointers to untyped ptr.
1712bool SPIRVEmitIntrinsics::postprocessTypes() {
1713 bool Changed = false;
1714 if (!GR)
1715 return Changed;
1716 for (auto IB = PostprocessWorklist.rbegin(), IE = PostprocessWorklist.rend();
1717 IB != IE; ++IB) {
1718 CallInst *AssignCI = GR->findAssignPtrTypeInstr(*IB);
1719 Type *KnownTy = GR->findDeducedElementType(*IB);
1720 if (!KnownTy || !AssignCI || !isa<Instruction>(AssignCI->getArgOperand(0)))
1721 continue;
1722 Instruction *I = cast<Instruction>(AssignCI->getArgOperand(0));
1723 for (User *U : I->users()) {
1724 Instruction *Inst = dyn_cast<Instruction>(U);
1725 if (!Inst || isa<IntrinsicInst>(Inst))
1726 continue;
1727 deduceOperandElementType(Inst, I, KnownTy, AssignCI);
1728 if (KnownTy != GR->findDeducedElementType(I)) {
1729 Changed = true;
1730 break;
1731 }
1732 }
1733 }
1734 return Changed;
1735}
1736
1737bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
1738 bool Changed = false;
1739
1740 UncompleteTypeInfo.clear();
1741 PostprocessWorklist.clear();
1742 for (auto &F : M)
1743 Changed |= runOnFunction(F);
1744
1745 for (auto &F : M) {
1746 // check if function parameter types are set
1747 if (!F.isDeclaration() && !F.isIntrinsic()) {
1748 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
1749 GR = ST.getSPIRVGlobalRegistry();
1750 IRBuilder<> B(F.getContext());
1751 processParamTypes(&F, B);
1752 }
1753 }
1754
1755 Changed |= postprocessTypes();
1756
1757 return Changed;
1758}
1759
1761 return new SPIRVEmitIntrinsics(TM);
1762}
static unsigned getIntrinsicID(const SDNode *N)
aarch64 promote const
unsigned Intr
always inline
Expand Atomic instructions
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
return RetTy
static bool runOnFunction(Function &F, bool PostInlining)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
uint64_t IntrinsicInst * II
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrConstForceInt32(const Value *V)
static Type * getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I, Value *PointerOperand)
static void reportFatalOnTokenType(const Instruction *I)
static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I)
static Type * getPointeeTypeByCallInst(StringRef DemangledName, Function *CalledF, unsigned OpIdx)
static Type * reconstructType(SPIRVGlobalRegistry *GR, Value *Op)
static bool IsKernelArgInt8(Function *F, StoreInst *SI)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static Type * restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I, Type *Ty)
static bool requireAssignType(Instruction *I)
static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB)
DEMANGLE_NAMESPACE_BEGIN bool starts_with(std::string_view self, char C) noexcept
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
an instruction to allocate memory on the stack
Definition: Instructions.h:61
Represent the analysis usage information of a pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:495
static unsigned getPointerOperandIndex()
Definition: Instructions.h:623
static unsigned getPointerOperandIndex()
Definition: Instructions.h:854
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:168
This class represents a no-op cast from one type to another.
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Definition: Constants.cpp:1871
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1532
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1415
unsigned arg_size() const
Definition: InstrTypes.h:1408
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This class represents an Operation in the Expression.
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:254
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:380
size_t arg_size() const
Definition: Function.h:899
Argument * getArg(unsigned i) const
Definition: Function.h:884
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:915
static unsigned getPointerOperandIndex()
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2686
Indirect Branch Instruction.
void addDestination(BasicBlock *Dest)
Add a destination.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
Definition: InstVisitor.h:78
RetTy visitExtractElementInst(ExtractElementInst &I)
Definition: InstVisitor.h:191
RetTy visitInsertValueInst(InsertValueInst &I)
Definition: InstVisitor.h:195
RetTy visitUnreachableInst(UnreachableInst &I)
Definition: InstVisitor.h:241
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
Definition: InstVisitor.h:171
RetTy visitBitCastInst(BitCastInst &I)
Definition: InstVisitor.h:187
RetTy visitSwitchInst(SwitchInst &I)
Definition: InstVisitor.h:232
RetTy visitExtractValueInst(ExtractValueInst &I)
Definition: InstVisitor.h:194
RetTy visitStoreInst(StoreInst &I)
Definition: InstVisitor.h:170
RetTy visitInsertElementInst(InsertElementInst &I)
Definition: InstVisitor.h:192
RetTy visitAllocaInst(AllocaInst &I)
Definition: InstVisitor.h:168
RetTy visitCallInst(CallInst &I)
Definition: InstVisitor.h:220
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
Definition: InstVisitor.h:174
void visitInstruction(Instruction &I)
Definition: InstVisitor.h:280
RetTy visitLoadInst(LoadInst &I)
Definition: InstVisitor.h:169
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:169
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:174
static unsigned getPointerOperandIndex()
Definition: Instructions.h:255
Metadata node.
Definition: Metadata.h:1069
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1542
A single uniqued string.
Definition: Metadata.h:720
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:606
Flags
Flags values. These may be or'd together.
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
static MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition: Metadata.cpp:103
Metadata * getMetadata() const
Definition: Metadata.h:193
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:37
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1852
void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI)
Type * findDeducedCompositeType(const Value *Val)
void addDeducedElementType(Value *Val, Type *Ty)
void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy)
Type * findMutated(const Value *Val)
void addDeducedCompositeType(Value *Val, Type *Ty)
Type * findDeducedElementType(const Value *Val)
CallInst * findAssignPtrTypeInstr(const Value *Val)
bool canUseExtension(SPIRV::Extension::Extension E) const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:442
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:503
size_t size() const
Definition: SmallVector.h:92
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
An instruction for storing to memory.
Definition: Instructions.h:290
static unsigned getPointerOperandIndex()
Definition: Instructions.h:379
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:128
iterator end()
Definition: StringMap.h:220
iterator find(StringRef Key)
Definition: StringMap.h:233
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:250
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:501
Multiway switch.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
Definition: DerivedTypes.h:720
static TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types=std::nullopt, ArrayRef< unsigned > Ints=std::nullopt)
Return a target extension type having the specified name and optional type and integer parameters.
Definition: Type.cpp:784
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:261
StringRef getTargetExtName() const
bool isTargetExtTy() const
Return true if this is a target extension type.
Definition: Type.h:203
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:291
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
A few GPU targets, such as DXIL and SPIR-V, have typed pointers.
static TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
This function has undefined behavior.
op_range operands()
Definition: User.h:242
Value * getOperand(unsigned i) const
Definition: User.h:169
static ConstantAsMetadata * getConstant(Value *C)
Definition: Metadata.h:472
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:255
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
bool user_empty() const
Definition: Value.h:385
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ SPIR_KERNEL
Used for SPIR kernel functions.
Definition: CallingConv.h:144
std::tuple< int, unsigned, unsigned > mapBuiltinToOpcode(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set)
Helper function for finding a builtin function attributes by a demangled function name.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
std::string lookupBuiltinNameHelper(StringRef DemangledCall)
Parses the name part of the demangled builtin call.
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &)
ModulePass * createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM)
unsigned getPointerAddressSpace(const Type *T)
Definition: SPIRVUtils.h:126
AddressSpace
Definition: NVPTXBaseInfo.h:21
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)
Definition: SPIRVUtils.cpp:339
bool isTypedPointerTy(const Type *T)
Definition: SPIRVUtils.h:110
Type * getTypedPointerWrapper(Type *ElemTy, unsigned AS)
Definition: SPIRVUtils.h:157
bool isPointerTy(const Type *T)
Definition: SPIRVUtils.h:120
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
@ Ref
The access may reference the value stored in memory.
DWARFExpression::Operation Op
Type * getPointeeTypeByAttr(Argument *Arg)
Definition: SPIRVUtils.h:139
bool hasPointeeTypeAttr(Argument *Arg)
Definition: SPIRVUtils.h:134
bool isEquivalentTypes(Type *Ty1, Type *Ty2)
Definition: SPIRVUtils.h:203
Type * getPointeeType(Type *Ty)
Definition: SPIRVUtils.h:182
void addStringImm(const StringRef &Str, MCInst &Inst)
Definition: SPIRVUtils.cpp:51
bool isUntypedPointerTy(const Type *T)
Definition: SPIRVUtils.h:115
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition: SPIRVUtils.cpp:236