LLVM 23.0.0git
SPIRVInstructionSelector.cpp
Go to the documentation of this file.
1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVTypeInst.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
27#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
28#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
29#include "llvm/CodeGen/MachineInstrBuilder.h"
30#include "llvm/CodeGen/MachineRegisterInfo.h"
31#include "llvm/CodeGen/Register.h"
32#include "llvm/CodeGen/TargetOpcodes.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/IntrinsicsSPIRV.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/ErrorHandling.h"
37#include <optional>
38
39#define DEBUG_TYPE "spirv-isel"
40
41using namespace llvm;
42namespace CL = SPIRV::OpenCLExtInst;
43namespace GL = SPIRV::GLSLExtInst;
44
45using ExtInstList =
46 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
47
48namespace {
49
50struct ImageOperands {
51 std::optional<Register> Bias;
52 std::optional<Register> Offset;
53 std::optional<Register> MinLod;
54 std::optional<Register> GradX;
55 std::optional<Register> GradY;
56 std::optional<Register> Lod;
57 std::optional<Register> Compare;
58};
59
60struct SplitParts {
61 SPIRVTypeInst Type = nullptr;
62 Register High;
63 Register Low;
64 bool IsScalar = false;
65};
66
67llvm::SPIRV::SelectionControl::SelectionControl
68getSelectionOperandForImm(int Imm) {
69 if (Imm == 2)
70 return SPIRV::SelectionControl::Flatten;
71 if (Imm == 1)
72 return SPIRV::SelectionControl::DontFlatten;
73 if (Imm == 0)
74 return SPIRV::SelectionControl::None;
75 llvm_unreachable("Invalid immediate");
76}
77
78#define GET_GLOBALISEL_PREDICATE_BITSET
79#include "SPIRVGenGlobalISel.inc"
80#undef GET_GLOBALISEL_PREDICATE_BITSET
81
82class SPIRVInstructionSelector : public InstructionSelector {
83 const SPIRVSubtarget &STI;
84 const SPIRVInstrInfo &TII;
85 const SPIRVRegisterInfo &TRI;
86 const RegisterBankInfo &RBI;
87 SPIRVGlobalRegistry &GR;
88 MachineRegisterInfo *MRI;
89 MachineFunction *HasVRegsReset = nullptr;
90
91 /// We need to keep track of the number we give to anonymous global values to
92 /// generate the same name every time when this is needed.
93 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
94 SmallPtrSet<MachineInstr *, 8> DeadMIs;
95
96public:
97 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
98 const SPIRVSubtarget &ST,
99 const RegisterBankInfo &RBI);
100 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
101 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
102 BlockFrequencyInfo *BFI) override;
103 // Common selection code. Instruction-specific selection occurs in spvSelect.
104 bool select(MachineInstr &I) override;
105 static const char *getName() { return DEBUG_TYPE; }
106
107#define GET_GLOBALISEL_PREDICATES_DECL
108#include "SPIRVGenGlobalISel.inc"
109#undef GET_GLOBALISEL_PREDICATES_DECL
110
111#define GET_GLOBALISEL_TEMPORARIES_DECL
112#include "SPIRVGenGlobalISel.inc"
113#undef GET_GLOBALISEL_TEMPORARIES_DECL
114
115private:
116 void resetVRegsType(MachineFunction &MF);
117 void removeDeadInstruction(MachineInstr &MI) const;
118 void removeOpNamesForDeadMI(MachineInstr &MI) const;
119
120 // tblgen-erated 'select' implementation, used as the initial selector for
121 // the patterns that don't require complex C++.
122 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
123
124 // All instruction-specific selection that didn't happen in "select()".
125 // Is basically a large Switch/Case delegating to all other select method.
126 bool spvSelect(Register ResVReg, SPIRVTypeInst ResType,
127 MachineInstr &I) const;
128
129 bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType,
130 MachineInstr &I, bool IsSigned) const;
131
132 bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType,
133 MachineInstr &I) const;
134
135 bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType,
136 MachineInstr &I, unsigned ExtendOpcode,
137 unsigned BitSetOpcode) const;
138
139 bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType,
140 MachineInstr &I, Register SrcReg,
141 unsigned BitSetOpcode) const;
142
143 bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType,
144 MachineInstr &I, Register SrcReg,
145 unsigned BitSetOpcode, bool SwapPrimarySide) const;
146
147 bool selectFirstBitSet64Overflow(Register ResVReg, SPIRVTypeInst ResType,
148 MachineInstr &I, Register SrcReg,
149 unsigned BitSetOpcode,
150 bool SwapPrimarySide) const;
151
152 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
153 const MachineInstr *Init = nullptr) const;
154
155 bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType,
156 MachineInstr &I, std::vector<Register> SrcRegs,
157 unsigned Opcode) const;
158
159 bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
160 unsigned Opcode) const;
161
162 bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType,
163 MachineInstr &I) const;
164
165 bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
166 MachineInstr &I) const;
167 bool selectStore(MachineInstr &I) const;
168
169 bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
170 MachineInstr &I) const;
171 bool selectStackRestore(MachineInstr &I) const;
172
173 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
174 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
175 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
176 bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
177
178 bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
179 unsigned NewOpcode, unsigned NegateOpcode = 0) const;
180
181 bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType,
182 MachineInstr &I) const;
183
184 bool selectFence(MachineInstr &I) const;
185
186 bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType,
187 MachineInstr &I) const;
188
189 bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
190 unsigned OpType) const;
191
192 bool selectAll(Register ResVReg, SPIRVTypeInst ResType,
193 MachineInstr &I) const;
194
195 bool selectAny(Register ResVReg, SPIRVTypeInst ResType,
196 MachineInstr &I) const;
197
198 bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType,
199 MachineInstr &I) const;
200
201 bool selectBitreverse16(Register ResVReg, SPIRVTypeInst ResType,
202 MachineInstr &I, Register Op) const;
203
204 bool selectBitreverseNative(Register ResVReg, SPIRVTypeInst ResType,
205 MachineInstr &I, Register Op) const;
206
207 bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType,
208 MachineInstr &I) const;
209 bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType,
210 MachineInstr &I) const;
211
212 bool selectCmp(Register ResVReg, SPIRVTypeInst ResType,
213 unsigned comparisonOpcode, MachineInstr &I) const;
214 bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType,
215 MachineInstr &I) const;
216
217 bool selectICmp(Register ResVReg, SPIRVTypeInst ResType,
218 MachineInstr &I) const;
219 bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType,
220 MachineInstr &I) const;
221
222 bool selectSign(Register ResVReg, SPIRVTypeInst ResType,
223 MachineInstr &I) const;
224
225 bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType,
226 MachineInstr &I) const;
227
228 bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType,
229 MachineInstr &I, unsigned Opcode) const;
230 bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType,
231 MachineInstr &I) const;
232
233 bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType,
234 MachineInstr &I, bool Signed) const;
235
236 bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType,
237 MachineInstr &I) const;
238
239 bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType,
240 MachineInstr &I) const;
241
242 bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType,
243 MachineInstr &I) const;
244
245 bool selectPopCount(Register ResVReg, SPIRVTypeInst ResType,
246 MachineInstr &I, unsigned Opcode) const;
247
248 bool selectPopCount16(Register ResVReg, SPIRVTypeInst ResType,
249 MachineInstr &I, unsigned ExtOpcode,
250 unsigned Opcode) const;
251
252 bool selectPopCount32(Register ResVReg, SPIRVTypeInst ResType,
253 MachineInstr &I, Register SrcReg,
254 unsigned Opcode) const;
255
256 bool selectPopCount64(Register ResVReg, SPIRVTypeInst ResType,
257 MachineInstr &I, Register SrcReg,
258 unsigned Opcode) const;
259
260 bool selectPopCount64Overflow(Register ResVReg, SPIRVTypeInst ResType,
261 MachineInstr &I, Register SrcReg,
262 unsigned Opcode) const;
263
264 template <bool Signed>
265 bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType,
266 MachineInstr &I) const;
267 template <bool Signed>
268 bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType,
269 MachineInstr &I) const;
270
271 bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType,
272 MachineInstr &I) const;
273
274 template <typename PickOpcodeFn>
275 bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType,
276 MachineInstr &I, bool IsUnsigned,
277 PickOpcodeFn &&PickOpcode) const;
278
279 bool selectWaveReduceOp(Register ResVReg, SPIRVTypeInst ResType,
280 MachineInstr &I, unsigned Opcode) const;
281
282 bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType,
283 MachineInstr &I, bool IsUnsigned) const;
284
285 bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType,
286 MachineInstr &I, bool IsUnsigned) const;
287
288 bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType,
289 MachineInstr &I) const;
290
291 bool selectWaveReduceProduct(Register ResVReg, const SPIRVTypeInst ResType,
292 MachineInstr &I) const;
293
294 template <typename PickOpcodeFn>
295 bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType,
296 MachineInstr &I, bool IsUnsigned,
297 PickOpcodeFn &&PickOpcode) const;
298
299 bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType,
300 MachineInstr &I) const;
301
302 bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
303 MachineInstr &I) const;
304
305 bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
306 unsigned Direction) const;
307
308 bool selectConst(Register ResVReg, SPIRVTypeInst ResType,
309 MachineInstr &I) const;
310
311 bool selectSelect(Register ResVReg, SPIRVTypeInst ResType,
312 MachineInstr &I) const;
313 bool selectBoolToInt(Register ResVReg, SPIRVTypeInst ResType,
314 Register BooleanVReg, MachineInstr &InsertAt,
315 bool IsSigned) const;
316 bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
317 bool IsSigned, unsigned Opcode) const;
318 bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
319 bool IsSigned) const;
320
321 bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType,
322 MachineInstr &I) const;
323
324 bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
325 bool IsSigned) const;
326
327 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
328 SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const;
329
330 bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType,
331 MachineInstr &I) const;
332 bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType,
333 MachineInstr &I) const;
334 bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType,
335 MachineInstr &I) const;
336 bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType,
337 MachineInstr &I) const;
338 bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType,
339 MachineInstr &I) const;
340 bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType,
341 MachineInstr &I) const;
342 bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType,
343 MachineInstr &I) const;
344 bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
345 MachineInstr &I) const;
346
347 bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
348 MachineInstr &I) const;
349 bool selectMaskedScatter(MachineInstr &I) const;
350
351 bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const;
352
353 bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
354 MachineInstr &I) const;
355 bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
356 MachineInstr &I) const;
357
358 bool selectBranch(MachineInstr &I) const;
359 bool selectBranchCond(MachineInstr &I) const;
360
361 bool selectPhi(Register ResVReg, MachineInstr &I) const;
362
363 bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I,
364 GL::GLSLExtInst GLInst, bool setMIFlags = true,
365 bool useMISrc = true,
366 ArrayRef<Register> SrcRegs = {}) const;
367 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
368 CL::OpenCLExtInst CLInst, bool setMIFlags = true,
369 bool useMISrc = true,
370 ArrayRef<Register> SrcRegs = {}) const;
371 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
372 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
373 bool setMIFlags = true, bool useMISrc = true,
374 ArrayRef<Register> SrcRegs = {}) const;
375 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
376 const ExtInstList &ExtInsts, bool setMIFlags = true,
377 bool useMISrc = true,
378 ArrayRef<Register> SrcRegs = {}) const;
379
380 bool selectLog10(Register ResVReg, SPIRVTypeInst ResType,
381 MachineInstr &I) const;
382
383 bool selectFpowi(Register ResVReg, SPIRVTypeInst ResType,
384 MachineInstr &I) const;
385
386 bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType,
387 MachineInstr &I) const;
388
389 bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType,
390 MachineInstr &I, unsigned Opcode) const;
391
392 bool selectBarrierInst(MachineInstr &I, unsigned Scope, unsigned MemSem,
393 bool WithGroupSync) const;
394
395 bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType,
396 MachineInstr &I) const;
397
398 bool selectWaveActiveAllEqual(Register ResVReg, SPIRVTypeInst ResType,
399 MachineInstr &I) const;
400
401 bool selectUnmergeValues(MachineInstr &I) const;
402
403 bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
404 MachineInstr &I) const;
405
406 bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
407 MachineInstr &I) const;
408
409 bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
410 MachineInstr &I) const;
411 bool selectGetDimensionsIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
412 MachineInstr &I) const;
413 bool selectGetDimensionsLevelsIntrinsic(Register &ResVReg,
414 SPIRVTypeInst ResType,
415 MachineInstr &I) const;
416 bool selectGetDimensionsMSIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
417 MachineInstr &I) const;
418 bool
419 selectImageQuerySize(Register ImageReg, Register &ResVReg, MachineInstr &I,
420 std::optional<Register> LodReg = std::nullopt) const;
421 bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
422 MachineInstr &I) const;
423 bool selectCalculateLodIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
424 MachineInstr &I) const;
425 bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
426 MachineInstr &I) const;
427 bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
428 MachineInstr &I) const;
429 bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
430 MachineInstr &I) const;
431 bool selectLoadLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
432 MachineInstr &I) const;
433 bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
434 MachineInstr &I) const;
435 bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg,
436 SPIRVTypeInst ResType,
437 MachineInstr &I) const;
438 bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
439 MachineInstr &I) const;
440 bool selectImageWriteIntrinsic(MachineInstr &I) const;
441 bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
442 MachineInstr &I) const;
443 bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
444 MachineInstr &I) const;
445 bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType,
446 MachineInstr &I) const;
447 bool selectModf(Register ResVReg, SPIRVTypeInst ResType,
448 MachineInstr &I) const;
449 bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType,
450 MachineInstr &I) const;
451 bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType,
452 MachineInstr &I) const;
453 bool selectSincos(Register ResVReg, SPIRVTypeInst ResType,
454 MachineInstr &I) const;
455 bool selectExp10(Register ResVReg, SPIRVTypeInst ResType,
456 MachineInstr &I) const;
457 bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType,
458 MachineInstr &I, const unsigned DPdOpCode) const;
459 // Utilities
460 Register buildI32Constant(uint32_t Val, MachineInstr &I,
461 SPIRVTypeInst ResType = nullptr) const;
462
463 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const;
464 bool isScalarOrVectorIntConstantZero(Register Reg) const;
465 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const;
466 Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
467 MachineInstr &I) const;
468 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
469
470 bool wrapIntoSpecConstantOp(MachineInstr &I,
471 SmallVector<Register> &CompositeArgs) const;
472
473 Register getUcharPtrTypeReg(MachineInstr &I,
474 SPIRV::StorageClass::StorageClass SC) const;
475 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
476 Register Src, Register DestType,
477 uint32_t Opcode) const;
478 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
479 SPIRVTypeInst SrcPtrTy) const;
480 Register buildPointerToResource(SPIRVTypeInst ResType,
481 SPIRV::StorageClass::StorageClass SC,
482 uint32_t Set, uint32_t Binding,
483 uint32_t ArraySize, Register IndexReg,
484 StringRef Name,
485 MachineIRBuilder MIRBuilder) const;
486 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const;
487 bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType,
488 Register &ReadReg, MachineInstr &InsertionPoint) const;
489 bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType,
490 Register ImageReg, Register IdxReg,
491 DebugLoc Loc, MachineInstr &Pos,
492 const ImageOperands *ImOps = nullptr) const;
493 bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType,
494 Register ImageReg, Register SamplerReg,
495 Register CoordinateReg, const ImageOperands &ImOps,
496 DebugLoc Loc, MachineInstr &I) const;
497 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
498 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
499 Register ResVReg, SPIRVTypeInst ResType,
500 MachineInstr &I) const;
501 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
502 Register ResVReg, SPIRVTypeInst ResType,
503 MachineInstr &I) const;
504 bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType,
505 GIntrinsic &HandleDef, MachineInstr &Pos) const;
506 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
507 void errorIfInstrOutsideShader(MachineInstr &I) const;
508
509 std::optional<SplitParts> splitEvenOddLanes(Register PopCountReg,
510 unsigned ComponentCount,
511 MachineInstr &I,
512 SPIRVTypeInst I32Type) const;
513};
514
515bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
516 const TargetExtType *TET = cast<TargetExtType>(HandleType);
517 if (TET->getTargetExtName() == "spirv.Image") {
518 return false;
519 }
520 assert(TET->getTargetExtName() == "spirv.SignedImage");
521 return TET->getTypeParameter(0)->isIntegerTy();
522}
523} // end anonymous namespace
524
525#define GET_GLOBALISEL_IMPL
526#include "SPIRVGenGlobalISel.inc"
527#undef GET_GLOBALISEL_IMPL
528
529SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
530 const SPIRVSubtarget &ST,
531 const RegisterBankInfo &RBI)
532 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
533 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
534 MRI(nullptr),
535#define GET_GLOBALISEL_PREDICATES_INIT
536#include "SPIRVGenGlobalISel.inc"
537#undef GET_GLOBALISEL_PREDICATES_INIT
538#define GET_GLOBALISEL_TEMPORARIES_INIT
539#include "SPIRVGenGlobalISel.inc"
540#undef GET_GLOBALISEL_TEMPORARIES_INIT
541{
542}
543
544void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
545 GISelValueTracking *VT,
546 CodeGenCoverage *CoverageInfo,
547 ProfileSummaryInfo *PSI,
548 BlockFrequencyInfo *BFI) {
549 MRI = &MF.getRegInfo();
550 GR.setCurrentFunc(MF);
551 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
552}
553
554// Ensure that register classes correspond to pattern matching rules.
555void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
556 if (HasVRegsReset == &MF)
557 return;
558 HasVRegsReset = &MF;
559
560 MachineRegisterInfo &MRI = MF.getRegInfo();
561 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
562 Register Reg = Register::index2VirtReg(I);
563 LLT RegType = MRI.getType(Reg);
564 if (RegType.isScalar())
565 MRI.setType(Reg, LLT::scalar(64));
566 else if (RegType.isPointer())
567 MRI.setType(Reg, LLT::pointer(0, 64));
568 else if (RegType.isVector())
569 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64)));
570 }
571 for (const auto &MBB : MF) {
572 for (const auto &MI : MBB) {
573 if (isPreISelGenericOpcode(MI.getOpcode()))
574 GR.erase(&MI);
575 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
576 continue;
577
578 Register DstReg = MI.getOperand(0).getReg();
579 LLT DstType = MRI.getType(DstReg);
580 Register SrcReg = MI.getOperand(1).getReg();
581 LLT SrcType = MRI.getType(SrcReg);
582 if (DstType != SrcType)
583 MRI.setType(DstReg, MRI.getType(SrcReg));
584
585 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg);
586 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg);
587 if (DstRC != SrcRC && SrcRC)
588 MRI.setRegClass(DstReg, SrcRC);
589 }
590 }
591}
592
593// Return true if the MachineInstr represents a constant register
594static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef) {
595
596 SmallVector<MachineInstr *> Stack = {OpDef};
597 SmallPtrSet<MachineInstr *, 4> Visited;
598
599 while (!Stack.empty()) {
600 MachineInstr *MI = Stack.pop_back_val();
601 MI = passCopy(MI, MRI);
602 if (!Visited.insert(MI).second)
603 continue;
604 switch (MI->getOpcode()) {
605 case TargetOpcode::G_INTRINSIC:
606 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
607 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
608 GIntrinsic *GIntr = cast<GIntrinsic>(MI);
609 unsigned IntrID = GIntr->getIntrinsicID();
610 if (IntrID != Intrinsic::spv_const_composite &&
611 IntrID != Intrinsic::spv_undef)
612 return false;
613 continue;
614 }
615 case TargetOpcode::G_BUILD_VECTOR:
616 case TargetOpcode::G_SPLAT_VECTOR:
617 for (unsigned i = OpDef->getNumExplicitDefs();
618 i < OpDef->getNumOperands(); i++) {
619 if (!OpDef->getOperand(i).isReg())
620 continue;
621 MachineInstr *OpNestedDef =
622 MRI->getVRegDef(OpDef->getOperand(i).getReg());
623 Stack.push_back(OpNestedDef);
624 }
625 continue;
626 case TargetOpcode::G_CONSTANT:
627 case TargetOpcode::G_FCONSTANT:
628 case TargetOpcode::G_IMPLICIT_DEF:
629 case SPIRV::OpConstantTrue:
630 case SPIRV::OpConstantFalse:
631 case SPIRV::OpConstantI:
632 case SPIRV::OpConstantF:
633 case SPIRV::OpConstantComposite:
634 case SPIRV::OpConstantCompositeContinuedINTEL:
635 case SPIRV::OpConstantSampler:
636 case SPIRV::OpConstantNull:
637 case SPIRV::OpUndef:
638 case SPIRV::OpConstantFunctionPointerINTEL:
639 continue;
640 default:
641 return false;
642 }
643 }
644 return true;
645}
646
647// Return true if the virtual register represents a constant
648static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
649 if (MachineInstr *OpDef = MRI->getVRegDef(OpReg))
650 return isConstReg(MRI, OpDef);
651 return false;
652}
653
654// TODO(168736): We should make this either a flag in tabelgen
655// or reduce our dependence on the global registry, so we can remove this
656// function. It can easily be missed when new intrinsics are added.
657
658// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
659// definition because they are referenced in the global registry. This is a list
660// of intrinsics that have no side effects other than their references in the
661// global registry.
662static bool intrinsicHasSideEffects(Intrinsic::ID ID) {
663 switch (ID) {
664 // This is not an exhaustive list and may need to be updated.
665 case Intrinsic::spv_all:
666 case Intrinsic::spv_alloca:
667 case Intrinsic::spv_any:
668 case Intrinsic::spv_bitcast:
669 case Intrinsic::spv_const_composite:
670 case Intrinsic::spv_cross:
671 case Intrinsic::spv_degrees:
672 case Intrinsic::spv_distance:
673 case Intrinsic::spv_extractelt:
674 case Intrinsic::spv_extractv:
675 case Intrinsic::spv_faceforward:
676 case Intrinsic::spv_fdot:
677 case Intrinsic::spv_firstbitlow:
678 case Intrinsic::spv_firstbitshigh:
679 case Intrinsic::spv_firstbituhigh:
680 case Intrinsic::spv_frac:
681 case Intrinsic::spv_gep:
682 case Intrinsic::spv_global_offset:
683 case Intrinsic::spv_global_size:
684 case Intrinsic::spv_group_id:
685 case Intrinsic::spv_insertelt:
686 case Intrinsic::spv_insertv:
687 case Intrinsic::spv_isinf:
688 case Intrinsic::spv_isnan:
689 case Intrinsic::spv_lerp:
690 case Intrinsic::spv_length:
691 case Intrinsic::spv_normalize:
692 case Intrinsic::spv_num_subgroups:
693 case Intrinsic::spv_num_workgroups:
694 case Intrinsic::spv_ptrcast:
695 case Intrinsic::spv_radians:
696 case Intrinsic::spv_reflect:
697 case Intrinsic::spv_refract:
698 case Intrinsic::spv_resource_getpointer:
699 case Intrinsic::spv_resource_handlefrombinding:
700 case Intrinsic::spv_resource_handlefromimplicitbinding:
701 case Intrinsic::spv_resource_nonuniformindex:
702 case Intrinsic::spv_resource_sample:
703 case Intrinsic::spv_rsqrt:
704 case Intrinsic::spv_saturate:
705 case Intrinsic::spv_sdot:
706 case Intrinsic::spv_sign:
707 case Intrinsic::spv_smoothstep:
708 case Intrinsic::spv_step:
709 case Intrinsic::spv_subgroup_id:
710 case Intrinsic::spv_subgroup_local_invocation_id:
711 case Intrinsic::spv_subgroup_max_size:
712 case Intrinsic::spv_subgroup_size:
713 case Intrinsic::spv_thread_id:
714 case Intrinsic::spv_thread_id_in_group:
715 case Intrinsic::spv_udot:
716 case Intrinsic::spv_undef:
717 case Intrinsic::spv_value_md:
718 case Intrinsic::spv_workgroup_size:
719 return false;
720 default:
721 return true;
722 }
723}
724
725// TODO(168736): We should make this either a flag in tabelgen
726// or reduce our dependence on the global registry, so we can remove this
727// function. It can easily be missed when new intrinsics are added.
728static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
729 switch (Opcode) {
730 case SPIRV::OpTypeVoid:
731 case SPIRV::OpTypeBool:
732 case SPIRV::OpTypeInt:
733 case SPIRV::OpTypeFloat:
734 case SPIRV::OpTypeVector:
735 case SPIRV::OpTypeMatrix:
736 case SPIRV::OpTypeImage:
737 case SPIRV::OpTypeSampler:
738 case SPIRV::OpTypeSampledImage:
739 case SPIRV::OpTypeArray:
740 case SPIRV::OpTypeRuntimeArray:
741 case SPIRV::OpTypeStruct:
742 case SPIRV::OpTypeOpaque:
743 case SPIRV::OpTypePointer:
744 case SPIRV::OpTypeFunction:
745 case SPIRV::OpTypeEvent:
746 case SPIRV::OpTypeDeviceEvent:
747 case SPIRV::OpTypeReserveId:
748 case SPIRV::OpTypeQueue:
749 case SPIRV::OpTypePipe:
750 case SPIRV::OpTypeForwardPointer:
751 case SPIRV::OpTypePipeStorage:
752 case SPIRV::OpTypeNamedBarrier:
753 case SPIRV::OpTypeAccelerationStructureNV:
754 case SPIRV::OpTypeCooperativeMatrixNV:
755 case SPIRV::OpTypeCooperativeMatrixKHR:
756 return true;
757 default:
758 return false;
759 }
760}
761
762bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
763 // If there are no definitions, then assume there is some other
764 // side-effect that makes this instruction live.
765 if (MI.getNumDefs() == 0)
766 return false;
767
768 for (const auto &MO : MI.all_defs()) {
769 Register Reg = MO.getReg();
770 if (Reg.isPhysical()) {
771 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
772 return false;
773 }
774 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
775 if (UseMI.getOpcode() != SPIRV::OpName) {
776 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
777 return false;
778 }
779 }
780 }
781
782 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
783 MI.isLifetimeMarker()) {
784 LLVM_DEBUG(
785 dbgs()
786 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
787 return false;
788 }
789 if (MI.isPHI()) {
790 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
791 return true;
792 }
793
794 // It is possible that the only side effect is that the instruction is
795 // referenced in the global registry. If that is the only side effect, the
796 // intrinsic is dead.
797 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
798 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
799 const auto &Intr = cast<GIntrinsic>(MI);
800 if (!intrinsicHasSideEffects(Intr.getIntrinsicID())) {
801 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
802 return true;
803 }
804 }
805
806 if (MI.mayStore() || MI.isCall() ||
807 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
808 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
809 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
810 return false;
811 }
812
813 if (isPreISelGenericOpcode(MI.getOpcode())) {
814 // TODO: Is there a generic way to check if the opcode has side effects?
815 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
816 return true;
817 }
818
819 if (isOpcodeWithNoSideEffects(MI.getOpcode())) {
820 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
821 return true;
822 }
823
824 return false;
825}
826
827void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
828 // Delete the OpName that uses the result if there is one.
829 for (const auto &MO : MI.all_defs()) {
830 Register Reg = MO.getReg();
831 if (Reg.isPhysical())
832 continue;
833 SmallVector<MachineInstr *, 4> UselessOpNames;
834 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
835 assert(UseMI.getOpcode() == SPIRV::OpName &&
836 "There is still a use of the dead function.");
837 UselessOpNames.push_back(&UseMI);
838 }
839 for (MachineInstr *OpNameMI : UselessOpNames) {
840 GR.invalidateMachineInstr(OpNameMI);
841 OpNameMI->eraseFromParent();
842 }
843 }
844}
845
846void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
847 salvageDebugInfo(*MRI, MI);
848 GR.invalidateMachineInstr(&MI);
849 removeOpNamesForDeadMI(MI);
850 MI.eraseFromParent();
851}
852
853bool SPIRVInstructionSelector::select(MachineInstr &I) {
854 resetVRegsType(*I.getParent()->getParent());
855
856 assert(I.getParent() && "Instruction should be in a basic block!");
857 assert(I.getParent()->getParent() && "Instruction should be in a function!");
858
859 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
860 if (isDead(I, *MRI)) {
861 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
862 removeDeadInstruction(I);
863 return true;
864 }
865
866 Register Opcode = I.getOpcode();
867 // If it's not a GMIR instruction, we've selected it already.
868 if (!isPreISelGenericOpcode(Opcode)) {
869 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
870 Register DstReg = I.getOperand(0).getReg();
871 Register SrcReg = I.getOperand(1).getReg();
872 auto *Def = MRI->getVRegDef(SrcReg);
873 if (isTypeFoldingSupported(Def->getOpcode()) &&
874 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
875 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
876 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
877 Register SelectDstReg = Def->getOperand(0).getReg();
878 bool SuccessToSelectSelect [[maybe_unused]] = selectSelect(
879 SelectDstReg, GR.getSPIRVTypeForVReg(SelectDstReg), *Def);
880 assert(SuccessToSelectSelect);
881 GR.invalidateMachineInstr(Def);
882 Def->eraseFromParent();
883 MRI->replaceRegWith(DstReg, SelectDstReg);
884 GR.invalidateMachineInstr(&I);
885 I.eraseFromParent();
886 return true;
887 }
888
889 bool Res = selectImpl(I, *CoverageInfo);
890 LLVM_DEBUG({
891 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
892 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
893 I.print(dbgs());
894 }
895 });
896 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
897 if (Res) {
898 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI))
899 DeadMIs.insert(Def);
900 return Res;
901 }
902 }
903 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg));
904 MRI->replaceRegWith(SrcReg, DstReg);
905 GR.invalidateMachineInstr(&I);
906 I.eraseFromParent();
907 return true;
908 } else if (I.getNumDefs() == 1) {
909 // Make all vregs 64 bits (for SPIR-V IDs).
910 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64));
911 }
912 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
913 return true;
914 }
915
916 if (DeadMIs.contains(&I)) {
917 // if the instruction has been already made dead by folding it away
918 // erase it
919 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
920 removeDeadInstruction(I);
921 DeadMIs.erase(&I);
922 return true;
923 }
924
925 if (I.getNumOperands() != I.getNumExplicitOperands()) {
926 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
927 return false;
928 }
929
930 // Common code for getting return reg+type, and removing selected instr
931 // from parent occurs here. Instr-specific selection happens in spvSelect().
932 bool HasDefs = I.getNumDefs() > 0;
933 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
934 SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
935 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
936 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
937 if (spvSelect(ResVReg, ResType, I)) {
938 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
939 for (unsigned i = 0; i < I.getNumDefs(); ++i)
940 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64));
941 GR.invalidateMachineInstr(&I);
942 I.eraseFromParent();
943 return true;
944 }
945 return false;
946}
947
948static bool mayApplyGenericSelection(unsigned Opcode) {
949 switch (Opcode) {
950 case TargetOpcode::G_CONSTANT:
951 case TargetOpcode::G_FCONSTANT:
952 return false;
953 }
954 return isTypeFoldingSupported(Opcode);
955}
956
957bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
958 MachineInstr &I) const {
959 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg);
960 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
961 if (DstRC != SrcRC && SrcRC)
962 MRI->setRegClass(DestReg, SrcRC);
963 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
964 .addDef(DestReg)
965 .addUse(SrcReg)
966 .constrainAllUses(TII, TRI, RBI);
967 return true;
968}
969
970bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
971 SPIRVTypeInst ResType,
972 MachineInstr &I) const {
973 const unsigned Opcode = I.getOpcode();
974 if (mayApplyGenericSelection(Opcode))
975 return selectImpl(I, *CoverageInfo);
976 switch (Opcode) {
977 case TargetOpcode::G_CONSTANT:
978 case TargetOpcode::G_FCONSTANT:
979 return selectConst(ResVReg, ResType, I);
980 case TargetOpcode::G_GLOBAL_VALUE:
981 return selectGlobalValue(ResVReg, I);
982 case TargetOpcode::G_IMPLICIT_DEF:
983 return selectOpUndef(ResVReg, ResType, I);
984 case TargetOpcode::G_FREEZE:
985 return selectFreeze(ResVReg, ResType, I);
986
987 case TargetOpcode::G_INTRINSIC:
988 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
989 case TargetOpcode::G_INTRINSIC_CONVERGENT:
990 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
991 return selectIntrinsic(ResVReg, ResType, I);
992 case TargetOpcode::G_BITREVERSE:
993 return selectBitreverse(ResVReg, ResType, I);
994
995 case TargetOpcode::G_BUILD_VECTOR:
996 return selectBuildVector(ResVReg, ResType, I);
997 case TargetOpcode::G_SPLAT_VECTOR:
998 return selectSplatVector(ResVReg, ResType, I);
999
1000 case TargetOpcode::G_SHUFFLE_VECTOR: {
1001 MachineBasicBlock &BB = *I.getParent();
1002 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
1003 .addDef(ResVReg)
1004 .addUse(GR.getSPIRVTypeID(ResType))
1005 .addUse(I.getOperand(1).getReg())
1006 .addUse(I.getOperand(2).getReg());
1007 for (auto V : I.getOperand(3).getShuffleMask())
1008 MIB.addImm(V);
1009 MIB.constrainAllUses(TII, TRI, RBI);
1010 return true;
1011 }
1012 case TargetOpcode::G_MEMMOVE:
1013 case TargetOpcode::G_MEMCPY:
1014 case TargetOpcode::G_MEMSET:
1015 return selectMemOperation(ResVReg, I);
1016
1017 case TargetOpcode::G_ICMP:
1018 return selectICmp(ResVReg, ResType, I);
1019 case TargetOpcode::G_FCMP:
1020 return selectFCmp(ResVReg, ResType, I);
1021
1022 case TargetOpcode::G_FRAME_INDEX:
1023 return selectFrameIndex(ResVReg, ResType, I);
1024
1025 case TargetOpcode::G_LOAD:
1026 return selectLoad(ResVReg, ResType, I);
1027 case TargetOpcode::G_STORE:
1028 return selectStore(I);
1029
1030 case TargetOpcode::G_BR:
1031 return selectBranch(I);
1032 case TargetOpcode::G_BRCOND:
1033 return selectBranchCond(I);
1034
1035 case TargetOpcode::G_PHI:
1036 return selectPhi(ResVReg, I);
1037
1038 case TargetOpcode::G_FPTOSI:
1039 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1040 case TargetOpcode::G_FPTOUI:
1041 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1042
1043 case TargetOpcode::G_FPTOSI_SAT:
1044 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1045 case TargetOpcode::G_FPTOUI_SAT:
1046 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1047
1048 case TargetOpcode::G_SITOFP:
1049 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
1050 case TargetOpcode::G_UITOFP:
1051 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
1052
1053 case TargetOpcode::G_CTPOP:
1054 return selectPopCount(ResVReg, ResType, I, SPIRV::OpBitCount);
1055 case TargetOpcode::G_SMIN:
1056 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
1057 case TargetOpcode::G_UMIN:
1058 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
1059
1060 case TargetOpcode::G_SMAX:
1061 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
1062 case TargetOpcode::G_UMAX:
1063 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
1064
1065 case TargetOpcode::G_SCMP:
1066 return selectSUCmp(ResVReg, ResType, I, true);
1067 case TargetOpcode::G_UCMP:
1068 return selectSUCmp(ResVReg, ResType, I, false);
1069 case TargetOpcode::G_LROUND:
1070 case TargetOpcode::G_LLROUND: {
1071 Register regForLround =
1072 MRI->createVirtualRegister(MRI->getRegClass(ResVReg), "lround");
1073 MRI->setRegClass(regForLround, &SPIRV::iIDRegClass);
1074 GR.assignSPIRVTypeToVReg(GR.getSPIRVTypeForVReg(I.getOperand(1).getReg()),
1075 regForLround, *(I.getParent()->getParent()));
1076 selectExtInst(regForLround, GR.getSPIRVTypeForVReg(regForLround), I,
1077 CL::round, GL::Round, /* setMIFlags */ false);
1078 MachineBasicBlock &BB = *I.getParent();
1079 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConvertFToS))
1080 .addDef(ResVReg)
1081 .addUse(GR.getSPIRVTypeID(ResType))
1082 .addUse(regForLround);
1083 MIB.constrainAllUses(TII, TRI, RBI);
1084 return true;
1085 }
1086 case TargetOpcode::G_STRICT_FMA:
1087 case TargetOpcode::G_FMA: {
1088 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_fma)) {
1089 MachineBasicBlock &BB = *I.getParent();
1090 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFmaKHR))
1091 .addDef(ResVReg)
1092 .addUse(GR.getSPIRVTypeID(ResType))
1093 .addUse(I.getOperand(1).getReg())
1094 .addUse(I.getOperand(2).getReg())
1095 .addUse(I.getOperand(3).getReg())
1096 .setMIFlags(I.getFlags());
1097 MIB.constrainAllUses(TII, TRI, RBI);
1098 return true;
1099 }
1100 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
1101 }
1102
1103 case TargetOpcode::G_STRICT_FLDEXP:
1104 return selectExtInst(ResVReg, ResType, I, CL::ldexp);
1105
1106 case TargetOpcode::G_FPOW:
1107 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
1108 case TargetOpcode::G_FPOWI:
1109 return selectFpowi(ResVReg, ResType, I);
1110
1111 case TargetOpcode::G_FEXP:
1112 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
1113 case TargetOpcode::G_FEXP2:
1114 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
1115 case TargetOpcode::G_FEXP10:
1116 return selectExp10(ResVReg, ResType, I);
1117
1118 case TargetOpcode::G_FMODF:
1119 return selectModf(ResVReg, ResType, I);
1120 case TargetOpcode::G_FSINCOS:
1121 return selectSincos(ResVReg, ResType, I);
1122
1123 case TargetOpcode::G_FLOG:
1124 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
1125 case TargetOpcode::G_FLOG2:
1126 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
1127 case TargetOpcode::G_FLOG10:
1128 return selectLog10(ResVReg, ResType, I);
1129
1130 case TargetOpcode::G_FABS:
1131 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
1132 case TargetOpcode::G_ABS:
1133 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
1134
1135 case TargetOpcode::G_FMINNUM:
1136 case TargetOpcode::G_FMINIMUM:
1137 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
1138 case TargetOpcode::G_FMAXNUM:
1139 case TargetOpcode::G_FMAXIMUM:
1140 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
1141
1142 case TargetOpcode::G_FCOPYSIGN:
1143 return selectExtInst(ResVReg, ResType, I, CL::copysign);
1144
1145 case TargetOpcode::G_FCEIL:
1146 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
1147 case TargetOpcode::G_FFLOOR:
1148 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
1149
1150 case TargetOpcode::G_FCOS:
1151 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
1152 case TargetOpcode::G_FSIN:
1153 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
1154 case TargetOpcode::G_FTAN:
1155 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
1156 case TargetOpcode::G_FACOS:
1157 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos);
1158 case TargetOpcode::G_FASIN:
1159 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
1160 case TargetOpcode::G_FATAN:
1161 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
1162 case TargetOpcode::G_FATAN2:
1163 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2);
1164 case TargetOpcode::G_FCOSH:
1165 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
1166 case TargetOpcode::G_FSINH:
1167 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh);
1168 case TargetOpcode::G_FTANH:
1169 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh);
1170
1171 case TargetOpcode::G_STRICT_FSQRT:
1172 case TargetOpcode::G_FSQRT:
1173 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
1174
1175 case TargetOpcode::G_CTTZ:
1176 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1177 return selectExtInst(ResVReg, ResType, I, CL::ctz);
1178 case TargetOpcode::G_CTLZ:
1179 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1180 return selectExtInst(ResVReg, ResType, I, CL::clz);
1181
1182 case TargetOpcode::G_INTRINSIC_ROUND:
1183 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
1184 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1185 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1186 case TargetOpcode::G_INTRINSIC_TRUNC:
1187 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
1188 case TargetOpcode::G_FRINT:
1189 case TargetOpcode::G_FNEARBYINT:
1190 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1191
1192 case TargetOpcode::G_SMULH:
1193 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
1194 case TargetOpcode::G_UMULH:
1195 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
1196
1197 case TargetOpcode::G_SADDSAT:
1198 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
1199 case TargetOpcode::G_UADDSAT:
1200 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
1201 case TargetOpcode::G_SSUBSAT:
1202 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
1203 case TargetOpcode::G_USUBSAT:
1204 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
1205
1206 case TargetOpcode::G_FFREXP:
1207 return selectFrexp(ResVReg, ResType, I);
1208
1209 case TargetOpcode::G_UADDO:
1210 return selectOverflowArith(ResVReg, ResType, I,
1211 ResType->getOpcode() == SPIRV::OpTypeVector
1212 ? SPIRV::OpIAddCarryV
1213 : SPIRV::OpIAddCarryS);
1214 case TargetOpcode::G_USUBO:
1215 return selectOverflowArith(ResVReg, ResType, I,
1216 ResType->getOpcode() == SPIRV::OpTypeVector
1217 ? SPIRV::OpISubBorrowV
1218 : SPIRV::OpISubBorrowS);
1219 case TargetOpcode::G_UMULO:
1220 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
1221 case TargetOpcode::G_SMULO:
1222 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
1223
1224 case TargetOpcode::G_SEXT:
1225 return selectExt(ResVReg, ResType, I, true);
1226 case TargetOpcode::G_ANYEXT:
1227 case TargetOpcode::G_ZEXT:
1228 return selectExt(ResVReg, ResType, I, false);
1229 case TargetOpcode::G_TRUNC:
1230 return selectTrunc(ResVReg, ResType, I);
1231 case TargetOpcode::G_FPTRUNC:
1232 case TargetOpcode::G_FPEXT:
1233 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
1234
1235 case TargetOpcode::G_PTRTOINT:
1236 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
1237 case TargetOpcode::G_INTTOPTR:
1238 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
1239 case TargetOpcode::G_BITCAST:
1240 return selectBitcast(ResVReg, ResType, I);
1241 case TargetOpcode::G_ADDRSPACE_CAST:
1242 return selectAddrSpaceCast(ResVReg, ResType, I);
1243 case TargetOpcode::G_PTR_ADD: {
1244 // Currently, we get G_PTR_ADD only applied to global variables.
1245 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1246 Register GV = I.getOperand(1).getReg();
1247 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
1248 (void)II;
1249 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1250 (*II).getOpcode() == TargetOpcode::COPY ||
1251 (*II).getOpcode() == SPIRV::OpVariable) &&
1252 getImm(I.getOperand(2), MRI));
1253 // It may be the initialization of a global variable.
1254 bool IsGVInit = false;
1255 for (MachineRegisterInfo::use_instr_iterator
1256 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()),
1257 UseEnd = MRI->use_instr_end();
1258 UseIt != UseEnd; UseIt = std::next(UseIt)) {
1259 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1260 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1261 (*UseIt).getOpcode() == SPIRV::OpVariable) {
1262 IsGVInit = true;
1263 break;
1264 }
1265 }
1266 MachineBasicBlock &BB = *I.getParent();
1267 if (!IsGVInit) {
1268 SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(GV);
1269 SPIRVTypeInst GVPointeeType = GR.getPointeeType(GVType);
1270 SPIRVTypeInst ResPointeeType = GR.getPointeeType(ResType);
1271 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1272 // Build a new virtual register that is associated with the required
1273 // data type.
1274 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV));
1275 MRI->setRegClass(NewVReg, MRI->getRegClass(GV));
1276 // Having a correctly typed base we are ready to build the actually
1277 // required GEP. It may not be a constant though, because all Operands
1278 // of OpSpecConstantOp is to originate from other const instructions,
1279 // and only the AccessChain named opcodes accept a global OpVariable
1280 // instruction. We can't use an AccessChain opcode because of the type
1281 // mismatch between result and base types.
1282 if (!GR.isBitcastCompatible(ResType, GVType))
1283 report_fatal_error(
1284 "incompatible result and operand types in a bitcast");
1285 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1286 MachineInstrBuilder MIB =
1287 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
1288 .addDef(NewVReg)
1289 .addUse(ResTypeReg)
1290 .addUse(GV);
1291 MIB.constrainAllUses(TII, TRI, RBI);
1292 BuildMI(BB, I, I.getDebugLoc(),
1293 TII.get(STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain
1294 : SPIRV::OpInBoundsPtrAccessChain))
1295 .addDef(ResVReg)
1296 .addUse(ResTypeReg)
1297 .addUse(NewVReg)
1298 .addUse(I.getOperand(2).getReg())
1299 .constrainAllUses(TII, TRI, RBI);
1300 } else {
1301 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1302 .addDef(ResVReg)
1303 .addUse(GR.getSPIRVTypeID(ResType))
1304 .addImm(
1305 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1306 .addUse(GV)
1307 .addUse(I.getOperand(2).getReg())
1308 .constrainAllUses(TII, TRI, RBI);
1309 }
1310 return true;
1311 }
1312 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1313 // initialize a global variable with a constant expression (e.g., the test
1314 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1315 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
1316 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1317 .addDef(ResVReg)
1318 .addUse(GR.getSPIRVTypeID(ResType))
1319 .addImm(static_cast<uint32_t>(
1320 SPIRV::Opcode::InBoundsPtrAccessChain))
1321 .addUse(GV)
1322 .addUse(Idx)
1323 .addUse(I.getOperand(2).getReg());
1324 MIB.constrainAllUses(TII, TRI, RBI);
1325 return true;
1326 }
1327
1328 case TargetOpcode::G_ATOMICRMW_OR:
1329 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
1330 case TargetOpcode::G_ATOMICRMW_ADD:
1331 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
1332 case TargetOpcode::G_ATOMICRMW_AND:
1333 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
1334 case TargetOpcode::G_ATOMICRMW_MAX:
1335 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
1336 case TargetOpcode::G_ATOMICRMW_MIN:
1337 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
1338 case TargetOpcode::G_ATOMICRMW_SUB:
1339 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
1340 case TargetOpcode::G_ATOMICRMW_XOR:
1341 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
1342 case TargetOpcode::G_ATOMICRMW_UMAX:
1343 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
1344 case TargetOpcode::G_ATOMICRMW_UMIN:
1345 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
1346 case TargetOpcode::G_ATOMICRMW_XCHG:
1347 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
1348 case TargetOpcode::G_ATOMIC_CMPXCHG:
1349 return selectAtomicCmpXchg(ResVReg, ResType, I);
1350
1351 case TargetOpcode::G_ATOMICRMW_FADD:
1352 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
1353 case TargetOpcode::G_ATOMICRMW_FSUB:
1354 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1355 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
1356 ResType->getOpcode() == SPIRV::OpTypeVector
1357 ? SPIRV::OpFNegateV
1358 : SPIRV::OpFNegate);
1359 case TargetOpcode::G_ATOMICRMW_FMIN:
1360 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
1361 case TargetOpcode::G_ATOMICRMW_FMAX:
1362 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
1363
1364 case TargetOpcode::G_FENCE:
1365 return selectFence(I);
1366
1367 case TargetOpcode::G_STACKSAVE:
1368 return selectStackSave(ResVReg, ResType, I);
1369 case TargetOpcode::G_STACKRESTORE:
1370 return selectStackRestore(I);
1371
1372 case TargetOpcode::G_UNMERGE_VALUES:
1373 return selectUnmergeValues(I);
1374
1375 // Discard gen opcodes for intrinsics which we do not expect to actually
1376 // represent code after lowering or intrinsics which are not implemented but
1377 // should not crash when found in a customer's LLVM IR input.
1378 case TargetOpcode::G_TRAP:
1379 case TargetOpcode::G_UBSANTRAP:
1380 case TargetOpcode::DBG_LABEL:
1381 return true;
1382 case TargetOpcode::G_DEBUGTRAP:
1383 return selectDebugTrap(ResVReg, ResType, I);
1384
1385 default:
1386 return false;
1387 }
1388}
1389
1390bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1391 SPIRVTypeInst ResType,
1392 MachineInstr &I) const {
1393 unsigned Opcode = SPIRV::OpNop;
1394 MachineBasicBlock &BB = *I.getParent();
1395 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1396 .constrainAllUses(TII, TRI, RBI);
1397 return true;
1398}
1399
1400bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1401 SPIRVTypeInst ResType,
1402 MachineInstr &I,
1403 GL::GLSLExtInst GLInst,
1404 bool setMIFlags, bool useMISrc,
1405 ArrayRef<Register> SrcRegs) const {
1406 if (!STI.canUseExtInstSet(
1407 SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) {
1408 std::string DiagMsg;
1409 raw_string_ostream OS(DiagMsg);
1410 I.print(OS, true, false, false, false);
1411 DiagMsg += " is only supported with the GLSL extended instruction set.\n";
1412 report_fatal_error(DiagMsg.c_str(), false);
1413 }
1414 return selectExtInst(ResVReg, ResType, I,
1415 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}},
1416 setMIFlags, useMISrc, SrcRegs);
1417}
1418
1419bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1420 SPIRVTypeInst ResType,
1421 MachineInstr &I,
1422 CL::OpenCLExtInst CLInst,
1423 bool setMIFlags, bool useMISrc,
1424 ArrayRef<Register> SrcRegs) const {
1425 return selectExtInst(ResVReg, ResType, I,
1426 {{SPIRV::InstructionSet::OpenCL_std, CLInst}},
1427 setMIFlags, useMISrc, SrcRegs);
1428}
1429
1430bool SPIRVInstructionSelector::selectExtInst(
1431 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1432 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, bool setMIFlags,
1433 bool useMISrc, ArrayRef<Register> SrcRegs) const {
1434 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1435 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1436 return selectExtInst(ResVReg, ResType, I, ExtInsts, setMIFlags, useMISrc,
1437 SrcRegs);
1438}
1439
1440bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1441 SPIRVTypeInst ResType,
1442 MachineInstr &I,
1443 const ExtInstList &Insts,
1444 bool setMIFlags, bool useMISrc,
1445 ArrayRef<Register> SrcRegs) const {
1446
1447 for (const auto &[InstructionSet, Opcode] : Insts) {
1448 if (!STI.canUseExtInstSet(InstructionSet))
1449 continue;
1450 MachineBasicBlock &BB = *I.getParent();
1451 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1452 .addDef(ResVReg)
1453 .addUse(GR.getSPIRVTypeID(ResType))
1454 .addImm(static_cast<uint32_t>(InstructionSet))
1455 .addImm(Opcode);
1456 if (setMIFlags)
1457 MIB.setMIFlags(I.getFlags());
1458 if (useMISrc) {
1459 const unsigned NumOps = I.getNumOperands();
1460 unsigned Index = 1;
1461 if (Index < NumOps &&
1462 I.getOperand(Index).getType() ==
1463 MachineOperand::MachineOperandType::MO_IntrinsicID)
1464 Index = 2;
1465 for (; Index < NumOps; ++Index)
1466 MIB.add(I.getOperand(Index));
1467 } else {
1468 for (Register SReg : SrcRegs) {
1469 MIB.addUse(SReg);
1470 }
1471 }
1472 MIB.constrainAllUses(TII, TRI, RBI);
1473 return true;
1474 }
1475 return false;
1476}
1477
1478bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1479 SPIRVTypeInst ResType,
1480 MachineInstr &I) const {
1481 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1482 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1483 for (const auto &Ex : ExtInsts) {
1484 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1485 uint32_t Opcode = Ex.second;
1486 if (!STI.canUseExtInstSet(Set))
1487 continue;
1488
1489 MachineIRBuilder MIRBuilder(I);
1490 SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(I.getOperand(1).getReg());
1491 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1492 PointeeTy, MIRBuilder, SPIRV::StorageClass::Function);
1493 Register PointerVReg =
1494 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1495
1496 auto It = getOpVariableMBBIt(I);
1497 BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
1498 .addDef(PointerVReg)
1499 .addUse(GR.getSPIRVTypeID(PointerType))
1500 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
1501 .constrainAllUses(TII, TRI, RBI);
1502
1503 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1504 .addDef(ResVReg)
1505 .addUse(GR.getSPIRVTypeID(ResType))
1506 .addImm(static_cast<uint32_t>(Ex.first))
1507 .addImm(Opcode)
1508 .add(I.getOperand(2))
1509 .addUse(PointerVReg)
1510 .constrainAllUses(TII, TRI, RBI);
1511
1512 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1513 .addDef(I.getOperand(1).getReg())
1514 .addUse(GR.getSPIRVTypeID(PointeeTy))
1515 .addUse(PointerVReg)
1516 .constrainAllUses(TII, TRI, RBI);
1517 return true;
1518 }
1519 return false;
1520}
1521
1522bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
1523 SPIRVTypeInst ResType,
1524 MachineInstr &I) const {
1525 Register CosResVReg = I.getOperand(1).getReg();
1526 unsigned SrcIdx = I.getNumExplicitDefs();
1527 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1528
1529 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1530 // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr.
1531 MachineIRBuilder MIRBuilder(I);
1532 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1533 ResType, MIRBuilder, SPIRV::StorageClass::Function);
1534 Register PointerVReg =
1535 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1536
1537 auto It = getOpVariableMBBIt(I);
1538 BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
1539 .addDef(PointerVReg)
1540 .addUse(GR.getSPIRVTypeID(PointerType))
1541 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
1542 .constrainAllUses(TII, TRI, RBI);
1543 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1544 .addDef(ResVReg)
1545 .addUse(ResTypeReg)
1546 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1547 .addImm(CL::sincos)
1548 .add(I.getOperand(SrcIdx))
1549 .addUse(PointerVReg)
1550 .constrainAllUses(TII, TRI, RBI);
1551 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1552 .addDef(CosResVReg)
1553 .addUse(ResTypeReg)
1554 .addUse(PointerVReg)
1555 .constrainAllUses(TII, TRI, RBI);
1556 return true;
1557 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
1558 // GLSL.std.450 has no combined sincos; emit separate Sin and Cos.
1559 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1560 .addDef(ResVReg)
1561 .addUse(ResTypeReg)
1562 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1563 .addImm(GL::Sin)
1564 .add(I.getOperand(SrcIdx))
1565 .constrainAllUses(TII, TRI, RBI);
1566 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1567 .addDef(CosResVReg)
1568 .addUse(ResTypeReg)
1569 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1570 .addImm(GL::Cos)
1571 .add(I.getOperand(SrcIdx))
1572 .constrainAllUses(TII, TRI, RBI);
1573 return true;
1574 }
1575 return false;
1576}
1577
1578bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1579 SPIRVTypeInst ResType,
1580 MachineInstr &I,
1581 std::vector<Register> Srcs,
1582 unsigned Opcode) const {
1583 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1584 .addDef(ResVReg)
1585 .addUse(GR.getSPIRVTypeID(ResType));
1586 for (Register SReg : Srcs) {
1587 MIB.addUse(SReg);
1588 }
1589 MIB.constrainAllUses(TII, TRI, RBI);
1590 return true;
1591}
1592
1593std::optional<SplitParts> SPIRVInstructionSelector::splitEvenOddLanes(
1594 Register PopCountReg, unsigned ComponentCount, MachineInstr &I,
1595 SPIRVTypeInst I32Type) const {
1596 SplitParts Parts;
1597
1598 if (ComponentCount == 1) {
1599 // ---- Scalar path: extract element 1 (high word) and element 0 (low word)
1600 // ----
1601 Parts.IsScalar = true;
1602 Parts.Type = I32Type;
1603 Parts.High = MRI->createVirtualRegister(GR.getRegClass(I32Type));
1604 Parts.Low = MRI->createVirtualRegister(GR.getRegClass(I32Type));
1605
1606 bool ZeroAsNull = !STI.isShader();
1607 Register IdxZero = GR.getOrCreateConstInt(0, I, I32Type, TII, ZeroAsNull);
1608 Register IdxOne = GR.getOrCreateConstInt(1, I, I32Type, TII, ZeroAsNull);
1609
1610 if (!selectOpWithSrcs(Parts.High, I32Type, I, {PopCountReg, IdxOne},
1611 SPIRV::OpVectorExtractDynamic))
1612 return std::nullopt;
1613
1614 if (!selectOpWithSrcs(Parts.Low, I32Type, I, {PopCountReg, IdxZero},
1615 SPIRV::OpVectorExtractDynamic))
1616 return std::nullopt;
1617
1618 } else {
1619 // ---- Vector path: shuffle odd lanes → High, even lanes → Low ----
1620 MachineIRBuilder MIRBuilder(I);
1621 Parts.IsScalar = false;
1622 Parts.Type = GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount,
1623 MIRBuilder, /*IsSigned=*/false);
1624 Parts.High = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1625 Parts.Low = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1626
1627 // High = odd-indexed elements (1, 3, 5, …) — the upper 32-bit halves.
1628 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1629 TII.get(SPIRV::OpVectorShuffle))
1630 .addDef(Parts.High)
1631 .addUse(GR.getSPIRVTypeID(Parts.Type))
1632 .addUse(PopCountReg)
1633 .addUse(PopCountReg);
1634 for (unsigned J = 1; J < ComponentCount * 2; J += 2)
1635 MIB.addImm(J);
1636 MIB.constrainAllUses(TII, TRI, RBI);
1637
1638 // Low = even-indexed elements (0, 2, 4, …) — the lower 32-bit halves.
1639 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1640 TII.get(SPIRV::OpVectorShuffle))
1641 .addDef(Parts.Low)
1642 .addUse(GR.getSPIRVTypeID(Parts.Type))
1643 .addUse(PopCountReg)
1644 .addUse(PopCountReg);
1645 for (unsigned J = 0; J < ComponentCount * 2; J += 2)
1646 MIB.addImm(J);
1647 MIB.constrainAllUses(TII, TRI, RBI);
1648 }
1649
1650 return Parts;
1651}
1652
1653bool SPIRVInstructionSelector::selectPopCount16(Register ResVReg,
1654 SPIRVTypeInst ResType,
1655 MachineInstr &I,
1656 unsigned ExtOpcode,
1657 unsigned Opcode) const {
1658 Register OpReg = I.getOperand(1).getReg();
1659 unsigned NumElems = GR.getScalarOrVectorComponentCount(OpReg);
1660
1661 MachineIRBuilder MIRBuilder(I);
1662 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1663 SPIRVTypeInst I32VectorType =
1664 GR.getOrCreateSPIRVVectorType(I32Type, NumElems, MIRBuilder, false);
1665
1666 bool IsVector = NumElems > 1;
1667 SPIRVTypeInst ExtType = IsVector ? I32VectorType : I32Type;
1668 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1669 // Always use OpUConvert to always use a 0 extend
1670 if (!selectOpWithSrcs(ExtReg, ExtType, I, {OpReg}, SPIRV::OpUConvert))
1671 return false;
1672
1673 Register PopCountReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1674 if (!selectPopCount32(PopCountReg, ExtType, I, ExtReg, Opcode))
1675 return false;
1676
1677 return selectOpWithSrcs(ResVReg, ResType, I, {PopCountReg}, ExtOpcode);
1678}
1679
1680bool SPIRVInstructionSelector::selectPopCount32(Register ResVReg,
1681 SPIRVTypeInst ResType,
1682 MachineInstr &I,
1683 Register SrcReg,
1684 unsigned Opcode) const {
1685 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
1686}
1687
1688bool SPIRVInstructionSelector::selectPopCount64Overflow(
1689 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
1690 unsigned int Opcode) const {
1691
1692 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
1693 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
1694
1695 MachineIRBuilder MIRBuilder(I);
1696 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
1697 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder);
1698 SPIRVTypeInst I64x2Type =
1699 GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder, false);
1700 SPIRVTypeInst Vec2ResType =
1701 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder, false);
1702
1703 std::vector<Register> PartialRegs;
1704
1705 unsigned CurrentComponent = 0;
1706 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
1707 Register PopCountResult =
1708 MRI->createVirtualRegister(GR.getRegClass(I64x2Type));
1709
1710 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1711 TII.get(SPIRV::OpVectorShuffle))
1712 .addDef(PopCountResult)
1713 .addUse(GR.getSPIRVTypeID(I64x2Type))
1714 .addUse(SrcReg)
1715 .addUse(SrcReg)
1716 .addImm(CurrentComponent)
1717 .addImm(CurrentComponent + 1);
1718
1719 MIB.constrainAllUses(TII, TRI, RBI);
1720
1721 Register SubVecReg =
1722 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType));
1723
1724 if (!selectPopCount64(SubVecReg, Vec2ResType, I, PopCountResult, Opcode))
1725 return false;
1726
1727 PartialRegs.push_back(SubVecReg);
1728 }
1729
1730 // On odd component counts we need to handle one more component
1731 if (CurrentComponent != ComponentCount) {
1732 bool ZeroAsNull = !STI.isShader();
1733 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
1734 Register ConstIntLastIdx = GR.getOrCreateConstInt(
1735 ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
1736
1737 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx},
1738 SPIRV::OpVectorExtractDynamic))
1739 return false;
1740
1741 Register FinalElemResReg =
1742 MRI->createVirtualRegister(GR.getRegClass(BaseType));
1743
1744 if (!selectPopCount64(FinalElemResReg, BaseType, I, FinalElemReg, Opcode))
1745 return false;
1746
1747 PartialRegs.push_back(FinalElemResReg);
1748 }
1749
1750 // Join all the resulting registers back into the return type in order
1751 // (ie i32x2, i32x2, i32x1 -> i32x5)
1752 return selectOpWithSrcs(ResVReg, ResType, I, std::move(PartialRegs),
1753 SPIRV::OpCompositeConstruct);
1754}
1755
1756bool SPIRVInstructionSelector::selectPopCount64(Register ResVReg,
1757 SPIRVTypeInst ResType,
1758 MachineInstr &I,
1759 Register SrcReg,
1760 unsigned Opcode) const {
1761 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
1762 if (ComponentCount > 2)
1763 return selectPopCount64Overflow(ResVReg, ResType, I, SrcReg, Opcode);
1764
1765 MachineIRBuilder MIRBuilder(I);
1766
1767 // ---- Types ----
1768 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1769 SPIRVTypeInst VecI32Type = GR.getOrCreateSPIRVVectorType(
1770 I32Type, 2 * ComponentCount, MIRBuilder, /*IsSigned=*/false);
1771
1772 // ---- Stage 1: 64-bit → 2x32-bit ----
1773 Register Vec32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
1774 if (!selectOpWithSrcs(Vec32, VecI32Type, I, {SrcReg}, SPIRV::OpBitcast))
1775 return false;
1776
1777 // ---- Stage 2: popcount per 32-bit lane ----
1778 Register Pop32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
1779 if (!selectPopCount32(Pop32, VecI32Type, I, Vec32, Opcode))
1780 return false;
1781
1782 // ---- Stage 3: split even (low) / odd (high) lanes ----
1783 auto MaybeParts = splitEvenOddLanes(Pop32, ComponentCount, I, I32Type);
1784 if (!MaybeParts)
1785 return false;
1786 SplitParts &Parts = *MaybeParts;
1787
1788 // ---- Stage 4: sum high + low ----
1789 unsigned OpAdd = Parts.IsScalar ? SPIRV::OpIAddS : SPIRV::OpIAddV;
1790 Register Sum = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1791 if (!selectOpWithSrcs(Sum, Parts.Type, I, {Parts.High, Parts.Low}, OpAdd))
1792 return false;
1793
1794 // ---- Stage 5: convert i32 sum to the final result type ----
1795 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1796 unsigned ConvOp = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1797 return selectOpWithSrcs(ResVReg, ResType, I, {Sum}, ConvOp);
1798}
1799
1800bool SPIRVInstructionSelector::selectPopCount(Register ResVReg,
1801 SPIRVTypeInst ResType,
1802 MachineInstr &I,
1803 unsigned Opcode) const {
1804 // Vulkan restricts OpBitCount to 32-bit integers or vectors of 32-bit
1805 // integers unless VK_KHR_maintenance9 is enabled. Until VK_KHR_maintaince9
1806 // is core we will not generate OpBitCount with any other types when
1807 // targeting Vulkan.
1808 if (!STI.getTargetTriple().isVulkanOS())
1809 return selectUnOp(ResVReg, ResType, I, Opcode);
1810
1811 Register OpReg = I.getOperand(1).getReg();
1812 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
1813 unsigned ExtOpcode = GR.isScalarOrVectorSigned(ResType) ? SPIRV::OpSConvert
1814 : SPIRV::OpUConvert;
1815 switch (GR.getScalarOrVectorBitWidth(OpType)) {
1816 case 8:
1817 case 16:
1818 return selectPopCount16(ResVReg, ResType, I, ExtOpcode, Opcode);
1819 case 32:
1820 return selectPopCount32(ResVReg, ResType, I, OpReg, Opcode);
1821 case 64:
1822 return selectPopCount64(ResVReg, ResType, I, OpReg, Opcode);
1823 default:
1824 report_fatal_error("unsupported operand bit width for popcount");
1825 }
1826}
1827
1828bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1829 SPIRVTypeInst ResType,
1830 MachineInstr &I,
1831 unsigned Opcode) const {
1832 if (STI.isPhysicalSPIRV() && I.getOperand(1).isReg()) {
1833 Register SrcReg = I.getOperand(1).getReg();
1834 bool IsGV = false;
1835 for (MachineRegisterInfo::def_instr_iterator DefIt =
1836 MRI->def_instr_begin(SrcReg);
1837 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
1838 unsigned DefOpCode = DefIt->getOpcode();
1839 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1840 // We need special handling to look through the type assignment or the
1841 // COPY pseudo-op and see if this is a constant or a global.
1842 if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg()))
1843 DefOpCode = VRD->getOpcode();
1844 }
1845 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1846 DefOpCode == TargetOpcode::G_CONSTANT ||
1847 DefOpCode == SPIRV::OpVariable || DefOpCode == SPIRV::OpConstantI) {
1848 IsGV = true;
1849 break;
1850 }
1851 }
1852 if (IsGV) {
1853 uint32_t SpecOpcode = 0;
1854 switch (Opcode) {
1855 case SPIRV::OpConvertPtrToU:
1856 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1857 break;
1858 case SPIRV::OpConvertUToPtr:
1859 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1860 break;
1861 }
1862 if (SpecOpcode) {
1863 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1864 TII.get(SPIRV::OpSpecConstantOp))
1865 .addDef(ResVReg)
1866 .addUse(GR.getSPIRVTypeID(ResType))
1867 .addImm(SpecOpcode)
1868 .addUse(SrcReg)
1869 .constrainAllUses(TII, TRI, RBI);
1870 return true;
1871 }
1872 }
1873 }
1874 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()},
1875 Opcode);
1876}
1877
1878bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1879 SPIRVTypeInst ResType,
1880 MachineInstr &I) const {
1881 Register OpReg = I.getOperand(1).getReg();
1882 SPIRVTypeInst OpType =
1883 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
1884 if (!GR.isBitcastCompatible(ResType, OpType))
1885 report_fatal_error("incompatible result and operand types in a bitcast");
1886 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
1887}
1888
1889static void addMemoryOperands(MachineMemOperand *MemOp,
1890 MachineInstrBuilder &MIB,
1891 MachineIRBuilder &MIRBuilder,
1892 SPIRVGlobalRegistry &GR) {
1893 const SPIRVSubtarget *ST =
1894 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1895 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1896 if (MemOp->isVolatile())
1897 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1898 if (MemOp->isNonTemporal())
1899 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1900 // Aligned memory operand requires the Kernel capability.
1901 if (!ST->isShader() && MemOp->getAlign().value())
1902 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1903
1904 [[maybe_unused]] MachineInstr *AliasList = nullptr;
1905 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
1906 if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
1907 if (auto *MD = MemOp->getAAInfo().Scope) {
1908 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
1909 if (AliasList)
1910 SpvMemOp |=
1911 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
1912 }
1913 if (auto *MD = MemOp->getAAInfo().NoAlias) {
1914 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
1915 if (NoAliasList)
1916 SpvMemOp |=
1917 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
1918 }
1919 }
1920
1921 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1922 MIB.addImm(SpvMemOp);
1923 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1924 MIB.addImm(MemOp->getAlign().value());
1925 if (AliasList)
1926 MIB.addUse(AliasList->getOperand(0).getReg());
1927 if (NoAliasList)
1928 MIB.addUse(NoAliasList->getOperand(0).getReg());
1929 }
1930}
1931
1932static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
1933 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1934 if (Flags & MachineMemOperand::Flags::MOVolatile)
1935 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1936 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1937 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1938
1939 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1940 MIB.addImm(SpvMemOp);
1941}
1942
1943bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1944 SPIRVTypeInst ResType,
1945 MachineInstr &I) const {
1946 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
1947 Register Ptr = I.getOperand(1 + OpOffset).getReg();
1948
1949 auto *PtrDef = getVRegDef(*MRI, Ptr);
1950 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
1951 if (IntPtrDef &&
1952 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1953 Register HandleReg = IntPtrDef->getOperand(2).getReg();
1954 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
1955 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1956 Register NewHandleReg =
1957 MRI->createVirtualRegister(MRI->getRegClass(HandleReg));
1958 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
1959 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
1960 return false;
1961 }
1962
1963 Register IdxReg = IntPtrDef->getOperand(3).getReg();
1964 return generateImageReadOrFetch(ResVReg, ResType, NewHandleReg, IdxReg,
1965 I.getDebugLoc(), I);
1966 }
1967 }
1968
1969 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1970 .addDef(ResVReg)
1971 .addUse(GR.getSPIRVTypeID(ResType))
1972 .addUse(Ptr);
1973 if (!I.getNumMemOperands()) {
1974 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1975 I.getOpcode() ==
1976 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1977 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
1978 } else {
1979 MachineIRBuilder MIRBuilder(I);
1980 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
1981 }
1982 MIB.constrainAllUses(TII, TRI, RBI);
1983 return true;
1984}
1985
1986bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1987 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
1988 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
1989 Register Ptr = I.getOperand(1 + OpOffset).getReg();
1990
1991 auto *PtrDef = getVRegDef(*MRI, Ptr);
1992 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
1993 if (IntPtrDef &&
1994 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1995 Register HandleReg = IntPtrDef->getOperand(2).getReg();
1996 Register NewHandleReg =
1997 MRI->createVirtualRegister(MRI->getRegClass(HandleReg));
1998 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
1999 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
2000 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
2001 return false;
2002 }
2003
2004 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2005 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
2006 auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2007 TII.get(SPIRV::OpImageWrite))
2008 .addUse(NewHandleReg)
2009 .addUse(IdxReg)
2010 .addUse(StoreVal);
2011
2012 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType);
2013 if (sampledTypeIsSignedInteger(LLVMHandleType))
2014 BMI.addImm(0x1000); // SignExtend
2015
2016 BMI.constrainAllUses(TII, TRI, RBI);
2017 return true;
2018 }
2019 }
2020
2021 MachineBasicBlock &BB = *I.getParent();
2022 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
2023 .addUse(Ptr)
2024 .addUse(StoreVal);
2025 if (!I.getNumMemOperands()) {
2026 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2027 I.getOpcode() ==
2028 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2029 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
2030 } else {
2031 MachineIRBuilder MIRBuilder(I);
2032 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2033 }
2034 MIB.constrainAllUses(TII, TRI, RBI);
2035 return true;
2036}
2037
2038bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
2039 SPIRVTypeInst ResType,
2040 MachineInstr &I) const {
2041 assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
2042 // Operand indices:
2043 // 0: result (def)
2044 // 1: intrinsic ID
2045 // 2: vector of pointers
2046 // 3: alignment (i32 immediate)
2047 // 4: mask (vector of i1)
2048 // 5: passthru/fill value
2049 const Register PtrsReg = I.getOperand(2).getReg();
2050 const uint32_t Alignment = I.getOperand(3).getImm();
2051 const Register MaskReg = I.getOperand(4).getReg();
2052 const Register PassthruReg = I.getOperand(5).getReg();
2053 const Register AlignmentReg = buildI32Constant(Alignment, I);
2054
2055 MachineBasicBlock &BB = *I.getParent();
2056 auto MIB =
2057 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL))
2058 .addDef(ResVReg)
2059 .addUse(GR.getSPIRVTypeID(ResType))
2060 .addUse(PtrsReg)
2061 .addUse(AlignmentReg)
2062 .addUse(MaskReg)
2063 .addUse(PassthruReg);
2064 MIB.constrainAllUses(TII, TRI, RBI);
2065 return true;
2066}
2067
2068bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
2069 assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
2070 // Operand indices (no explicit defs):
2071 // 0: intrinsic ID
2072 // 1: value vector
2073 // 2: vector of pointers
2074 // 3: alignment (i32 immediate)
2075 // 4: mask (vector of i1)
2076 const Register ValuesReg = I.getOperand(1).getReg();
2077 const Register PtrsReg = I.getOperand(2).getReg();
2078 const uint32_t Alignment = I.getOperand(3).getImm();
2079 const Register MaskReg = I.getOperand(4).getReg();
2080 const Register AlignmentReg = buildI32Constant(Alignment, I);
2081 MachineBasicBlock &BB = *I.getParent();
2082
2083 auto MIB =
2084 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL))
2085 .addUse(PtrsReg)
2086 .addUse(AlignmentReg)
2087 .addUse(MaskReg)
2088 .addUse(ValuesReg);
2089 MIB.constrainAllUses(TII, TRI, RBI);
2090 return true;
2091}
2092
2093bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I,
2094 const Twine &Msg) const {
2095 const Function &F = I.getMF()->getFunction();
2096 F.getContext().diagnose(
2097 DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error));
2098 return false;
2099}
2100
2101bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
2102 SPIRVTypeInst ResType,
2103 MachineInstr &I) const {
2104 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2105 report_fatal_error(
2106 "llvm.stacksave intrinsic: this instruction requires the following "
2107 "SPIR-V extension: SPV_INTEL_variable_length_array",
2108 false);
2109 MachineBasicBlock &BB = *I.getParent();
2110 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
2111 .addDef(ResVReg)
2112 .addUse(GR.getSPIRVTypeID(ResType))
2113 .constrainAllUses(TII, TRI, RBI);
2114 return true;
2115}
2116
2117bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
2118 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2119 report_fatal_error(
2120 "llvm.stackrestore intrinsic: this instruction requires the following "
2121 "SPIR-V extension: SPV_INTEL_variable_length_array",
2122 false);
2123 if (!I.getOperand(0).isReg())
2124 return false;
2125 MachineBasicBlock &BB = *I.getParent();
2126 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
2127 .addUse(I.getOperand(0).getReg())
2128 .constrainAllUses(TII, TRI, RBI);
2129 return true;
2130}
2131
2132Register
2133SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
2134 MachineIRBuilder MIRBuilder(I);
2135 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
2136
2137 // TODO: check if we have such GV, add init, use buildGlobalVariable.
2138 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
2139 Function &CurFunction = GR.CurMF->getFunction();
2140 Type *LLVMArrTy =
2141 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
2142 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
2143 true, GlobalValue::InternalLinkage,
2144 Constant::getNullValue(LLVMArrTy));
2145
2146 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2147 Type *ArrTy = ArrayType::get(ValTy, Num);
2148 SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType(
2149 ArrTy, MIRBuilder, SPIRV::StorageClass::UniformConstant);
2150
2151 SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType(
2152 ArrTy, MIRBuilder, SPIRV::AccessQualifier::None, false);
2153
2154 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
2155 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvArrTy, TII);
2156
2157 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2158 auto MIBVar =
2159 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
2160 .addDef(VarReg)
2161 .addUse(GR.getSPIRVTypeID(VarTy))
2162 .addImm(SPIRV::StorageClass::UniformConstant)
2163 .addUse(Const);
2164 MIBVar.constrainAllUses(TII, TRI, RBI);
2165
2166 GR.add(GV, MIBVar);
2167 GR.addGlobalObject(GV, GR.CurMF, VarReg);
2168
2169 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
2170 return VarReg;
2171}
2172
2173bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
2174 Register SrcReg) const {
2175 MachineBasicBlock &BB = *I.getParent();
2176 Register DstReg = I.getOperand(0).getReg();
2177 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg);
2178 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg);
2179 if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy))
2180 report_fatal_error("OpCopyMemory requires operands to have the same type");
2181 uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI);
2182 SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy);
2183 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy);
2184 if (!LLVMPointeeTy)
2185 report_fatal_error(
2186 "Unable to determine pointee type size for OpCopyMemory");
2187 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
2188 if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy)))
2189 report_fatal_error(
2190 "OpCopyMemory requires the size to match the pointee type size");
2191 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemory))
2192 .addUse(DstReg)
2193 .addUse(SrcReg);
2194 if (I.getNumMemOperands()) {
2195 MachineIRBuilder MIRBuilder(I);
2196 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2197 }
2198 MIB.constrainAllUses(TII, TRI, RBI);
2199 return true;
2200}
2201
2202bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
2203 Register SrcReg) const {
2204 MachineBasicBlock &BB = *I.getParent();
2205 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
2206 .addUse(I.getOperand(0).getReg())
2207 .addUse(SrcReg)
2208 .addUse(I.getOperand(2).getReg());
2209 if (I.getNumMemOperands()) {
2210 MachineIRBuilder MIRBuilder(I);
2211 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2212 }
2213 MIB.constrainAllUses(TII, TRI, RBI);
2214 return true;
2215}
2216
2217bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
2218 MachineInstr &I) const {
2219 Register SrcReg = I.getOperand(1).getReg();
2220 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
2221 Register VarReg = getOrCreateMemSetGlobal(I);
2222 if (!VarReg.isValid())
2223 return false;
2224 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2225 SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType(
2226 ValTy, I, SPIRV::StorageClass::UniformConstant);
2227 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2228 if (!selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast))
2229 return false;
2230 }
2231 if (STI.isLogicalSPIRV()) {
2232 if (!selectCopyMemory(I, SrcReg))
2233 return false;
2234 } else {
2235 if (!selectCopyMemorySized(I, SrcReg))
2236 return false;
2237 }
2238 if (ResVReg.isValid() && ResVReg != I.getOperand(0).getReg())
2239 if (!BuildCOPY(ResVReg, I.getOperand(0).getReg(), I))
2240 return false;
2241 return true;
2242}
2243
2244bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
2245 SPIRVTypeInst ResType,
2246 MachineInstr &I,
2247 unsigned NewOpcode,
2248 unsigned NegateOpcode) const {
2249 assert(I.hasOneMemOperand());
2250 const MachineMemOperand *MemOp = *I.memoperands_begin();
2251 uint32_t Scope = static_cast<uint32_t>(getMemScope(
2252 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID()));
2253 Register ScopeReg = buildI32Constant(Scope, I);
2254
2255 Register Ptr = I.getOperand(1).getReg();
2256 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
2257 // auto ScSem =
2258 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
2259 AtomicOrdering AO = MemOp->getSuccessOrdering();
2260 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2261 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
2262
2263 Register ValueReg = I.getOperand(2).getReg();
2264 if (NegateOpcode != 0) {
2265 // Translation with negative value operand is requested
2266 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, MRI->getMF());
2267 if (!selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode))
2268 return false;
2269 ValueReg = TmpReg;
2270 }
2271
2272 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
2273 .addDef(ResVReg)
2274 .addUse(GR.getSPIRVTypeID(ResType))
2275 .addUse(Ptr)
2276 .addUse(ScopeReg)
2277 .addUse(MemSemReg)
2278 .addUse(ValueReg)
2279 .constrainAllUses(TII, TRI, RBI);
2280 return true;
2281}
2282
2283bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
2284 unsigned ArgI = I.getNumOperands() - 1;
2285 Register SrcReg =
2286 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
2287 SPIRVTypeInst SrcType =
2288 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
2289 if (!SrcType || SrcType->getOpcode() != SPIRV::OpTypeVector)
2290 report_fatal_error(
2291 "cannot select G_UNMERGE_VALUES with a non-vector argument");
2292
2293 SPIRVTypeInst ScalarType =
2294 GR.getSPIRVTypeForVReg(SrcType->getOperand(1).getReg());
2295 MachineBasicBlock &BB = *I.getParent();
2296 unsigned CurrentIndex = 0;
2297 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2298 Register ResVReg = I.getOperand(i).getReg();
2299 SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(ResVReg);
2300 if (!ResType) {
2301 LLT ResLLT = MRI->getType(ResVReg);
2302 assert(ResLLT.isValid());
2303 if (ResLLT.isVector()) {
2304 ResType = GR.getOrCreateSPIRVVectorType(
2305 ScalarType, ResLLT.getNumElements(), I, TII);
2306 } else {
2307 ResType = ScalarType;
2308 }
2309 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2310 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
2311 }
2312
2313 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
2314 Register UndefReg = GR.getOrCreateUndef(I, SrcType, TII);
2315 auto MIB =
2316 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
2317 .addDef(ResVReg)
2318 .addUse(GR.getSPIRVTypeID(ResType))
2319 .addUse(SrcReg)
2320 .addUse(UndefReg);
2321 unsigned NumElements = GR.getScalarOrVectorComponentCount(ResType);
2322 for (unsigned j = 0; j < NumElements; ++j) {
2323 MIB.addImm(CurrentIndex + j);
2324 }
2325 CurrentIndex += NumElements;
2326 MIB.constrainAllUses(TII, TRI, RBI);
2327 } else {
2328 auto MIB =
2329 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2330 .addDef(ResVReg)
2331 .addUse(GR.getSPIRVTypeID(ResType))
2332 .addUse(SrcReg)
2333 .addImm(CurrentIndex);
2334 CurrentIndex++;
2335 MIB.constrainAllUses(TII, TRI, RBI);
2336 }
2337 }
2338 return true;
2339}
2340
2341bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
2342 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
2343 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2344 Register MemSemReg = buildI32Constant(MemSem, I);
2345 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
2346 uint32_t Scope = static_cast<uint32_t>(
2347 getMemScope(GR.CurMF->getFunction().getContext(), Ord));
2348 Register ScopeReg = buildI32Constant(Scope, I);
2349 MachineBasicBlock &BB = *I.getParent();
2350 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
2351 .addUse(ScopeReg)
2352 .addUse(MemSemReg)
2353 .constrainAllUses(TII, TRI, RBI);
2354 return true;
2355}
2356
2357bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
2358 SPIRVTypeInst ResType,
2359 MachineInstr &I,
2360 unsigned Opcode) const {
2361 Type *ResTy = nullptr;
2362 StringRef ResName;
2363 if (!GR.findValueAttrs(&I, ResTy, ResName))
2364 report_fatal_error(
2365 "Not enough info to select the arithmetic with overflow instruction");
2366 if (!ResTy || !ResTy->isStructTy())
2367 report_fatal_error("Expect struct type result for the arithmetic "
2368 "with overflow instruction");
2369 // "Result Type must be from OpTypeStruct. The struct must have two members,
2370 // and the two members must be the same type."
2371 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
2372 ResTy = StructType::get(ResElemTy, ResElemTy);
2373 // Build SPIR-V types and constant(s) if needed.
2374 MachineIRBuilder MIRBuilder(I);
2375 SPIRVTypeInst StructType = GR.getOrCreateSPIRVType(
2376 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
2377 assert(I.getNumDefs() > 1 && "Not enought operands");
2378 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2379 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
2380 if (N > 1)
2381 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
2382 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
2383 Register ZeroReg = buildZerosVal(ResType, I);
2384 // A new virtual register to store the result struct.
2385 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2386 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
2387 // Build the result name if needed.
2388 if (ResName.size() > 0)
2389 buildOpName(StructVReg, ResName, MIRBuilder);
2390 // Build the arithmetic with overflow instruction.
2391 MachineBasicBlock &BB = *I.getParent();
2392 auto MIB =
2393 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
2394 .addDef(StructVReg)
2395 .addUse(GR.getSPIRVTypeID(StructType));
2396 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
2397 MIB.addUse(I.getOperand(i).getReg());
2398 MIB.constrainAllUses(TII, TRI, RBI);
2399 // Build instructions to extract fields of the instruction's result.
2400 // A new virtual register to store the higher part of the result struct.
2401 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2402 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
2403 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2404 auto MIB =
2405 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2406 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
2407 .addUse(GR.getSPIRVTypeID(ResType))
2408 .addUse(StructVReg)
2409 .addImm(i);
2410 MIB.constrainAllUses(TII, TRI, RBI);
2411 }
2412 // Build boolean value from the higher part.
2413 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
2414 .addDef(I.getOperand(1).getReg())
2415 .addUse(BoolTypeReg)
2416 .addUse(HigherVReg)
2417 .addUse(ZeroReg)
2418 .constrainAllUses(TII, TRI, RBI);
2419 return true;
2420}
2421
2422bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
2423 SPIRVTypeInst ResType,
2424 MachineInstr &I) const {
2425 Register ScopeReg;
2426 Register MemSemEqReg;
2427 Register MemSemNeqReg;
2428 Register Ptr = I.getOperand(2).getReg();
2429 if (!isa<GIntrinsic>(I)) {
2430 assert(I.hasOneMemOperand());
2431 const MachineMemOperand *MemOp = *I.memoperands_begin();
2432 unsigned Scope = static_cast<uint32_t>(getMemScope(
2433 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID()));
2434 ScopeReg = buildI32Constant(Scope, I);
2435
2436 unsigned ScSem = static_cast<uint32_t>(
2437 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
2438 AtomicOrdering AO = MemOp->getSuccessOrdering();
2439 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
2440 Register MemSemEqReg = buildI32Constant(MemSemEq, I);
2441 AtomicOrdering FO = MemOp->getFailureOrdering();
2442 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
2443 if (MemSemEq == MemSemNeq)
2444 MemSemNeqReg = MemSemEqReg;
2445 else {
2446 MemSemNeqReg = buildI32Constant(MemSemEq, I);
2447 }
2448 } else {
2449 ScopeReg = I.getOperand(5).getReg();
2450 MemSemEqReg = I.getOperand(6).getReg();
2451 MemSemNeqReg = I.getOperand(7).getReg();
2452 }
2453
2454 Register Cmp = I.getOperand(3).getReg();
2455 Register Val = I.getOperand(4).getReg();
2456 SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(Val);
2457 Register ACmpRes = createVirtualRegister(SpvValTy, &GR, MRI, *I.getMF());
2458 const DebugLoc &DL = I.getDebugLoc();
2459 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
2460 .addDef(ACmpRes)
2461 .addUse(GR.getSPIRVTypeID(SpvValTy))
2462 .addUse(Ptr)
2463 .addUse(ScopeReg)
2464 .addUse(MemSemEqReg)
2465 .addUse(MemSemNeqReg)
2466 .addUse(Val)
2467 .addUse(Cmp)
2468 .constrainAllUses(TII, TRI, RBI);
2469 SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2470 Register CmpSuccReg = createVirtualRegister(BoolTy, &GR, MRI, *I.getMF());
2471 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
2472 .addDef(CmpSuccReg)
2473 .addUse(GR.getSPIRVTypeID(BoolTy))
2474 .addUse(ACmpRes)
2475 .addUse(Cmp)
2476 .constrainAllUses(TII, TRI, RBI);
2477 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, *I.getMF());
2478 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2479 .addDef(TmpReg)
2480 .addUse(GR.getSPIRVTypeID(ResType))
2481 .addUse(ACmpRes)
2482 .addUse(GR.getOrCreateUndef(I, ResType, TII))
2483 .addImm(0)
2484 .constrainAllUses(TII, TRI, RBI);
2485 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2486 .addDef(ResVReg)
2487 .addUse(GR.getSPIRVTypeID(ResType))
2488 .addUse(CmpSuccReg)
2489 .addUse(TmpReg)
2490 .addImm(1)
2491 .constrainAllUses(TII, TRI, RBI);
2492 return true;
2493}
2494
2495static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2496 switch (SC) {
2497 case SPIRV::StorageClass::DeviceOnlyINTEL:
2498 case SPIRV::StorageClass::HostOnlyINTEL:
2499 return true;
2500 default:
2501 return false;
2502 }
2503}
2504
2505// Returns true ResVReg is referred only from global vars and OpName's.
2506static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
2507 bool IsGRef = false;
2508 bool IsAllowedRefs =
2509 llvm::all_of(MRI->use_instructions(ResVReg), [&IsGRef](auto const &It) {
2510 unsigned Opcode = It.getOpcode();
2511 if (Opcode == SPIRV::OpConstantComposite ||
2512 Opcode == SPIRV::OpSpecConstantComposite ||
2513 Opcode == SPIRV::OpVariable ||
2514 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2515 return IsGRef = true;
2516 return Opcode == SPIRV::OpName;
2517 });
2518 return IsAllowedRefs && IsGRef;
2519}
2520
2521Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2522 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2523 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType(
2524 Type::getInt8Ty(I.getMF()->getFunction().getContext()), I, SC));
2525}
2526
2527MachineInstrBuilder
2528SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2529 Register Src, Register DestType,
2530 uint32_t Opcode) const {
2531 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
2532 TII.get(SPIRV::OpSpecConstantOp))
2533 .addDef(Dest)
2534 .addUse(DestType)
2535 .addImm(Opcode)
2536 .addUse(Src);
2537}
2538
2539MachineInstrBuilder
2540SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2541 SPIRVTypeInst SrcPtrTy) const {
2542 SPIRVTypeInst GenericPtrTy =
2543 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
2544 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
2545 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace(
2546 SPIRV::StorageClass::Generic),
2547 GR.getPointerSize()));
2548 MachineFunction *MF = I.getParent()->getParent();
2549 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF);
2550 MachineInstrBuilder MIB = buildSpecConstantOp(
2551 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy),
2552 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
2553 GR.add(MIB.getInstr(), MIB);
2554 return MIB;
2555}
2556
2557// In SPIR-V address space casting can only happen to and from the Generic
2558// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
2559// pointers to and from Generic pointers. As such, we can convert e.g. from
2560// Workgroup to Function by going via a Generic pointer as an intermediary. All
2561// other combinations can only be done by a bitcast, and are probably not safe.
2562bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
2563 SPIRVTypeInst ResType,
2564 MachineInstr &I) const {
2565 MachineBasicBlock &BB = *I.getParent();
2566 const DebugLoc &DL = I.getDebugLoc();
2567
2568 Register SrcPtr = I.getOperand(1).getReg();
2569 SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
2570
2571 // don't generate a cast for a null that may be represented by OpTypeInt
2572 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
2573 ResType->getOpcode() != SPIRV::OpTypePointer)
2574 return BuildCOPY(ResVReg, SrcPtr, I);
2575
2576 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy);
2577 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType);
2578
2579 if (isASCastInGVar(MRI, ResVReg)) {
2580 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
2581 // are expressed by OpSpecConstantOp with an Opcode.
2582 // TODO: maybe insert a check whether the Kernel capability was declared and
2583 // so PtrCastToGeneric/GenericCastToPtr are available.
2584 unsigned SpecOpcode =
2585 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)
2586 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
2587 : (SrcSC == SPIRV::StorageClass::Generic &&
2588 isGenericCastablePtr(DstSC)
2589 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
2590 : 0);
2591 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
2592 // correct value of ResType and use general i8* instead. Maybe this should
2593 // be addressed in the emit-intrinsic step to infer a correct
2594 // OpConstantComposite type.
2595 if (SpecOpcode) {
2596 buildSpecConstantOp(I, ResVReg, SrcPtr, getUcharPtrTypeReg(I, DstSC),
2597 SpecOpcode)
2598 .constrainAllUses(TII, TRI, RBI);
2599 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
2600 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
2601 MIB.constrainAllUses(TII, TRI, RBI);
2602 buildSpecConstantOp(
2603 I, ResVReg, MIB->getOperand(0).getReg(), getUcharPtrTypeReg(I, DstSC),
2604 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
2605 .constrainAllUses(TII, TRI, RBI);
2606 }
2607 return true;
2608 }
2609
2610 // don't generate a cast between identical storage classes
2611 if (SrcSC == DstSC)
2612 return BuildCOPY(ResVReg, SrcPtr, I);
2613
2614 if ((SrcSC == SPIRV::StorageClass::Function &&
2615 DstSC == SPIRV::StorageClass::Private) ||
2616 (DstSC == SPIRV::StorageClass::Function &&
2617 SrcSC == SPIRV::StorageClass::Private))
2618 return BuildCOPY(ResVReg, SrcPtr, I);
2619
2620 // Casting from an eligible pointer to Generic.
2621 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
2622 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
2623 // Casting from Generic to an eligible pointer.
2624 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
2625 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
2626 // Casting between 2 eligible pointers using Generic as an intermediary.
2627 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
2628 SPIRVTypeInst GenericPtrTy =
2629 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
2630 Register Tmp = createVirtualRegister(GenericPtrTy, &GR, MRI, MRI->getMF());
2631 BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
2632 .addDef(Tmp)
2633 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
2634 .addUse(SrcPtr)
2635 .constrainAllUses(TII, TRI, RBI);
2636 BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
2637 .addDef(ResVReg)
2638 .addUse(GR.getSPIRVTypeID(ResType))
2639 .addUse(Tmp)
2640 .constrainAllUses(TII, TRI, RBI);
2641 return true;
2642 }
2643
2644 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
2645 // be applied
2646 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
2647 return selectUnOp(ResVReg, ResType, I,
2648 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
2649 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
2650 return selectUnOp(ResVReg, ResType, I,
2651 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
2652 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic)
2653 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
2654 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC))
2655 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
2656
2657 // Bitcast for pointers requires that the address spaces must match
2658 return false;
2659}
2660
2661static unsigned getFCmpOpcode(unsigned PredNum) {
2662 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2663 switch (Pred) {
2664 case CmpInst::FCMP_OEQ:
2665 return SPIRV::OpFOrdEqual;
2666 case CmpInst::FCMP_OGE:
2667 return SPIRV::OpFOrdGreaterThanEqual;
2668 case CmpInst::FCMP_OGT:
2669 return SPIRV::OpFOrdGreaterThan;
2670 case CmpInst::FCMP_OLE:
2671 return SPIRV::OpFOrdLessThanEqual;
2672 case CmpInst::FCMP_OLT:
2673 return SPIRV::OpFOrdLessThan;
2674 case CmpInst::FCMP_ONE:
2675 return SPIRV::OpFOrdNotEqual;
2676 case CmpInst::FCMP_ORD:
2677 return SPIRV::OpOrdered;
2678 case CmpInst::FCMP_UEQ:
2679 return SPIRV::OpFUnordEqual;
2680 case CmpInst::FCMP_UGE:
2681 return SPIRV::OpFUnordGreaterThanEqual;
2682 case CmpInst::FCMP_UGT:
2683 return SPIRV::OpFUnordGreaterThan;
2684 case CmpInst::FCMP_ULE:
2685 return SPIRV::OpFUnordLessThanEqual;
2686 case CmpInst::FCMP_ULT:
2687 return SPIRV::OpFUnordLessThan;
2688 case CmpInst::FCMP_UNE:
2689 return SPIRV::OpFUnordNotEqual;
2690 case CmpInst::FCMP_UNO:
2691 return SPIRV::OpUnordered;
2692 default:
2693 llvm_unreachable("Unknown predicate type for FCmp");
2694 }
2695}
2696
2697static unsigned getICmpOpcode(unsigned PredNum) {
2698 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2699 switch (Pred) {
2700 case CmpInst::ICMP_EQ:
2701 return SPIRV::OpIEqual;
2702 case CmpInst::ICMP_NE:
2703 return SPIRV::OpINotEqual;
2704 case CmpInst::ICMP_SGE:
2705 return SPIRV::OpSGreaterThanEqual;
2706 case CmpInst::ICMP_SGT:
2707 return SPIRV::OpSGreaterThan;
2708 case CmpInst::ICMP_SLE:
2709 return SPIRV::OpSLessThanEqual;
2710 case CmpInst::ICMP_SLT:
2711 return SPIRV::OpSLessThan;
2712 case CmpInst::ICMP_UGE:
2713 return SPIRV::OpUGreaterThanEqual;
2714 case CmpInst::ICMP_UGT:
2715 return SPIRV::OpUGreaterThan;
2716 case CmpInst::ICMP_ULE:
2717 return SPIRV::OpULessThanEqual;
2718 case CmpInst::ICMP_ULT:
2719 return SPIRV::OpULessThan;
2720 default:
2721 llvm_unreachable("Unknown predicate type for ICmp");
2722 }
2723}
2724
2725static unsigned getPtrCmpOpcode(unsigned Pred) {
2726 switch (static_cast<CmpInst::Predicate>(Pred)) {
2727 case CmpInst::ICMP_EQ:
2728 return SPIRV::OpPtrEqual;
2729 case CmpInst::ICMP_NE:
2730 return SPIRV::OpPtrNotEqual;
2731 default:
2732 llvm_unreachable("Unknown predicate type for pointer comparison");
2733 }
2734}
2735
2736// Return the logical operation, or abort if none exists.
2737static unsigned getBoolCmpOpcode(unsigned PredNum) {
2738 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2739 switch (Pred) {
2740 case CmpInst::ICMP_EQ:
2741 return SPIRV::OpLogicalEqual;
2742 case CmpInst::ICMP_NE:
2743 return SPIRV::OpLogicalNotEqual;
2744 default:
2745 llvm_unreachable("Unknown predicate type for Bool comparison");
2746 }
2747}
2748
2749static APFloat getZeroFP(const Type *LLVMFloatTy) {
2750 if (!LLVMFloatTy)
2751 return APFloat::getZero(APFloat::IEEEsingle());
2752 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2753 case Type::HalfTyID:
2754 return APFloat::getZero(APFloat::IEEEhalf());
2755 default:
2756 case Type::FloatTyID:
2757 return APFloat::getZero(APFloat::IEEEsingle());
2758 case Type::DoubleTyID:
2759 return APFloat::getZero(APFloat::IEEEdouble());
2760 }
2761}
2762
2763static APFloat getOneFP(const Type *LLVMFloatTy) {
2764 if (!LLVMFloatTy)
2765 return APFloat::getOne(APFloat::IEEEsingle());
2766 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2767 case Type::HalfTyID:
2768 return APFloat::getOne(APFloat::IEEEhalf());
2769 default:
2770 case Type::FloatTyID:
2771 return APFloat::getOne(APFloat::IEEEsingle());
2772 case Type::DoubleTyID:
2773 return APFloat::getOne(APFloat::IEEEdouble());
2774 }
2775}
2776
2777bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
2778 SPIRVTypeInst ResType,
2779 MachineInstr &I,
2780 unsigned OpAnyOrAll) const {
2781 assert(I.getNumOperands() == 3);
2782 assert(I.getOperand(2).isReg());
2783 MachineBasicBlock &BB = *I.getParent();
2784 Register InputRegister = I.getOperand(2).getReg();
2785 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
2786
2787 assert(InputType && "VReg has no type assigned");
2788
2789 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
2790 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
2791 if (IsBoolTy && !IsVectorTy) {
2792 assert(ResVReg == I.getOperand(0).getReg());
2793 return BuildCOPY(ResVReg, InputRegister, I);
2794 }
2795
2796 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
2797 unsigned SpirvNotEqualId =
2798 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
2799 SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
2800 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
2801 Register NotEqualReg = ResVReg;
2802
2803 if (IsVectorTy) {
2804 NotEqualReg =
2805 IsBoolTy ? InputRegister
2806 : createVirtualRegister(SpvBoolTy, &GR, MRI, MRI->getMF());
2807 const unsigned NumElts = InputType->getOperand(2).getImm();
2808 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
2809 }
2810
2811 if (!IsBoolTy) {
2812 Register ConstZeroReg =
2813 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
2814
2815 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
2816 .addDef(NotEqualReg)
2817 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
2818 .addUse(InputRegister)
2819 .addUse(ConstZeroReg)
2820 .constrainAllUses(TII, TRI, RBI);
2821 }
2822
2823 if (IsVectorTy)
2824 BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
2825 .addDef(ResVReg)
2826 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
2827 .addUse(NotEqualReg)
2828 .constrainAllUses(TII, TRI, RBI);
2829 return true;
2830}
2831
2832bool SPIRVInstructionSelector::selectAll(Register ResVReg,
2833 SPIRVTypeInst ResType,
2834 MachineInstr &I) const {
2835 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
2836}
2837
2838bool SPIRVInstructionSelector::selectAny(Register ResVReg,
2839 SPIRVTypeInst ResType,
2840 MachineInstr &I) const {
2841 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
2842}
2843
2844// Select the OpDot instruction for the given float dot
2845bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
2846 SPIRVTypeInst ResType,
2847 MachineInstr &I) const {
2848 assert(I.getNumOperands() == 4);
2849 assert(I.getOperand(2).isReg());
2850 assert(I.getOperand(3).isReg());
2851
2852 [[maybe_unused]] SPIRVTypeInst VecType =
2853 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg());
2854
2855 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2856 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2857 "dot product requires a vector of at least 2 components");
2858
2859 [[maybe_unused]] SPIRVTypeInst EltType =
2860 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg());
2861
2862 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
2863
2864 MachineBasicBlock &BB = *I.getParent();
2865 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot))
2866 .addDef(ResVReg)
2867 .addUse(GR.getSPIRVTypeID(ResType))
2868 .addUse(I.getOperand(2).getReg())
2869 .addUse(I.getOperand(3).getReg())
2870 .constrainAllUses(TII, TRI, RBI);
2871 return true;
2872}
2873
2874bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
2875 SPIRVTypeInst ResType,
2876 MachineInstr &I,
2877 bool Signed) const {
2878 assert(I.getNumOperands() == 4);
2879 assert(I.getOperand(2).isReg());
2880 assert(I.getOperand(3).isReg());
2881 MachineBasicBlock &BB = *I.getParent();
2882
2883 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2884 BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
2885 .addDef(ResVReg)
2886 .addUse(GR.getSPIRVTypeID(ResType))
2887 .addUse(I.getOperand(2).getReg())
2888 .addUse(I.getOperand(3).getReg())
2889 .constrainAllUses(TII, TRI, RBI);
2890 return true;
2891}
2892
2893// Since pre-1.6 SPIRV has no integer dot implementation,
2894// expand by piecewise multiplying and adding the results
2895bool SPIRVInstructionSelector::selectIntegerDotExpansion(
2896 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2897 assert(I.getNumOperands() == 4);
2898 assert(I.getOperand(2).isReg());
2899 assert(I.getOperand(3).isReg());
2900 MachineBasicBlock &BB = *I.getParent();
2901
2902 // Multiply the vectors, then sum the results
2903 Register Vec0 = I.getOperand(2).getReg();
2904 Register Vec1 = I.getOperand(3).getReg();
2905 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType));
2906 SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(Vec0);
2907
2908 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV))
2909 .addDef(TmpVec)
2910 .addUse(GR.getSPIRVTypeID(VecType))
2911 .addUse(Vec0)
2912 .addUse(Vec1)
2913 .constrainAllUses(TII, TRI, RBI);
2914
2915 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2916 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2917 "dot product requires a vector of at least 2 components");
2918
2919 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType));
2920 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2921 .addDef(Res)
2922 .addUse(GR.getSPIRVTypeID(ResType))
2923 .addUse(TmpVec)
2924 .addImm(0)
2925 .constrainAllUses(TII, TRI, RBI);
2926
2927 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) {
2928 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType));
2929
2930 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2931 .addDef(Elt)
2932 .addUse(GR.getSPIRVTypeID(ResType))
2933 .addUse(TmpVec)
2934 .addImm(i)
2935 .constrainAllUses(TII, TRI, RBI);
2936
2937 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1
2938 ? MRI->createVirtualRegister(GR.getRegClass(ResType))
2939 : ResVReg;
2940
2941 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
2942 .addDef(Sum)
2943 .addUse(GR.getSPIRVTypeID(ResType))
2944 .addUse(Res)
2945 .addUse(Elt)
2946 .constrainAllUses(TII, TRI, RBI);
2947 Res = Sum;
2948 }
2949
2950 return true;
2951}
2952
2953bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
2954 SPIRVTypeInst ResType,
2955 MachineInstr &I) const {
2956 MachineBasicBlock &BB = *I.getParent();
2957 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsInf))
2958 .addDef(ResVReg)
2959 .addUse(GR.getSPIRVTypeID(ResType))
2960 .addUse(I.getOperand(2).getReg())
2961 .constrainAllUses(TII, TRI, RBI);
2962 return true;
2963}
2964
2965bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
2966 SPIRVTypeInst ResType,
2967 MachineInstr &I) const {
2968 MachineBasicBlock &BB = *I.getParent();
2969 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan))
2970 .addDef(ResVReg)
2971 .addUse(GR.getSPIRVTypeID(ResType))
2972 .addUse(I.getOperand(2).getReg())
2973 .constrainAllUses(TII, TRI, RBI);
2974 return true;
2975}
2976
2977template <bool Signed>
2978bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
2979 SPIRVTypeInst ResType,
2980 MachineInstr &I) const {
2981 assert(I.getNumOperands() == 5);
2982 assert(I.getOperand(2).isReg());
2983 assert(I.getOperand(3).isReg());
2984 assert(I.getOperand(4).isReg());
2985 MachineBasicBlock &BB = *I.getParent();
2986
2987 Register Acc = I.getOperand(2).getReg();
2988 Register X = I.getOperand(3).getReg();
2989 Register Y = I.getOperand(4).getReg();
2990
2991 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2992 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType));
2993 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
2994 .addDef(Dot)
2995 .addUse(GR.getSPIRVTypeID(ResType))
2996 .addUse(X)
2997 .addUse(Y);
2998 MIB.addImm(SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
2999 MIB.constrainAllUses(TII, TRI, RBI);
3000
3001 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3002 .addDef(ResVReg)
3003 .addUse(GR.getSPIRVTypeID(ResType))
3004 .addUse(Dot)
3005 .addUse(Acc)
3006 .constrainAllUses(TII, TRI, RBI);
3007 return true;
3008}
3009
3010// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
3011// extract the elements of the packed inputs, multiply them and add the result
3012// to the accumulator.
3013template <bool Signed>
3014bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
3015 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3016 assert(I.getNumOperands() == 5);
3017 assert(I.getOperand(2).isReg());
3018 assert(I.getOperand(3).isReg());
3019 assert(I.getOperand(4).isReg());
3020 MachineBasicBlock &BB = *I.getParent();
3021
3022 Register Acc = I.getOperand(2).getReg();
3023 Register X = I.getOperand(3).getReg();
3024 Register Y = I.getOperand(4).getReg();
3025
3026 SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII);
3027 auto ExtractOp =
3028 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
3029
3030 bool ZeroAsNull = !STI.isShader();
3031 // Extract the i8 element, multiply and add it to the accumulator
3032 for (unsigned i = 0; i < 4; i++) {
3033 // A[i]
3034 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3035 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3036 .addDef(AElt)
3037 .addUse(GR.getSPIRVTypeID(ResType))
3038 .addUse(X)
3039 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3040 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3041 .constrainAllUses(TII, TRI, RBI);
3042
3043 // B[i]
3044 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3045 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3046 .addDef(BElt)
3047 .addUse(GR.getSPIRVTypeID(ResType))
3048 .addUse(Y)
3049 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3050 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3051 .constrainAllUses(TII, TRI, RBI);
3052
3053 // A[i] * B[i]
3054 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3055 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS))
3056 .addDef(Mul)
3057 .addUse(GR.getSPIRVTypeID(ResType))
3058 .addUse(AElt)
3059 .addUse(BElt)
3060 .constrainAllUses(TII, TRI, RBI);
3061
3062 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
3063 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3064 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3065 .addDef(MaskMul)
3066 .addUse(GR.getSPIRVTypeID(ResType))
3067 .addUse(Mul)
3068 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII, ZeroAsNull))
3069 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3070 .constrainAllUses(TII, TRI, RBI);
3071
3072 // Acc = Acc + A[i] * B[i]
3073 Register Sum =
3074 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg;
3075 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3076 .addDef(Sum)
3077 .addUse(GR.getSPIRVTypeID(ResType))
3078 .addUse(Acc)
3079 .addUse(MaskMul)
3080 .constrainAllUses(TII, TRI, RBI);
3081
3082 Acc = Sum;
3083 }
3084
3085 return true;
3086}
3087
3088/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
3089/// does not have a saturate builtin.
3090bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
3091 SPIRVTypeInst ResType,
3092 MachineInstr &I) const {
3093 assert(I.getNumOperands() == 3);
3094 assert(I.getOperand(2).isReg());
3095 MachineBasicBlock &BB = *I.getParent();
3096 Register VZero = buildZerosValF(ResType, I);
3097 Register VOne = buildOnesValF(ResType, I);
3098
3099 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
3100 .addDef(ResVReg)
3101 .addUse(GR.getSPIRVTypeID(ResType))
3102 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3103 .addImm(GL::FClamp)
3104 .addUse(I.getOperand(2).getReg())
3105 .addUse(VZero)
3106 .addUse(VOne)
3107 .constrainAllUses(TII, TRI, RBI);
3108 return true;
3109}
3110
3111bool SPIRVInstructionSelector::selectSign(Register ResVReg,
3112 SPIRVTypeInst ResType,
3113 MachineInstr &I) const {
3114 assert(I.getNumOperands() == 3);
3115 assert(I.getOperand(2).isReg());
3116 MachineBasicBlock &BB = *I.getParent();
3117 Register InputRegister = I.getOperand(2).getReg();
3118 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3119 auto &DL = I.getDebugLoc();
3120
3121 if (!InputType)
3122 report_fatal_error("Input Type could not be determined.");
3123
3124 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3125
3126 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType);
3127 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType);
3128
3129 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
3130
3131 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
3132 Register SignReg = NeedsConversion
3133 ? MRI->createVirtualRegister(&SPIRV::IDRegClass)
3134 : ResVReg;
3135
3136 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst))
3137 .addDef(SignReg)
3138 .addUse(GR.getSPIRVTypeID(InputType))
3139 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3140 .addImm(SignOpcode)
3141 .addUse(InputRegister)
3142 .constrainAllUses(TII, TRI, RBI);
3143
3144 if (NeedsConversion) {
3145 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
3146 BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode))
3147 .addDef(ResVReg)
3148 .addUse(GR.getSPIRVTypeID(ResType))
3149 .addUse(SignReg)
3150 .constrainAllUses(TII, TRI, RBI);
3151 }
3152
3153 return true;
3154}
3155
3156bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
3157 SPIRVTypeInst ResType,
3158 MachineInstr &I,
3159 unsigned Opcode) const {
3160 MachineBasicBlock &BB = *I.getParent();
3161 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3162
3163 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3164 .addDef(ResVReg)
3165 .addUse(GR.getSPIRVTypeID(ResType))
3166 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I,
3167 IntTy, TII, !STI.isShader()));
3168
3169 for (unsigned J = 2; J < I.getNumOperands(); J++) {
3170 BMI.addUse(I.getOperand(J).getReg());
3171 }
3172
3173 BMI.constrainAllUses(TII, TRI, RBI);
3174 return true;
3175}
3176
3177bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I,
3178 unsigned Scope,
3179 unsigned MemSem,
3180 bool WithGroupSync) const {
3181 auto BarrierType =
3182 WithGroupSync ? SPIRV::OpControlBarrier : SPIRV::OpMemoryBarrier;
3183
3184 MemSem |= SPIRV::MemorySemantics::AcquireRelease;
3185
3186 assert(((Scope != SPIRV::Scope::Workgroup) ||
3187 ((MemSem & SPIRV::MemorySemantics::WorkgroupMemory) > 0)) &&
3188 "Workgroup Scope must set WorkGroupMemory semantic "
3189 "in Barrier instruction");
3190
3191 assert(((Scope != SPIRV::Scope::Device) ||
3192 ((MemSem & SPIRV::MemorySemantics::UniformMemory) > 0 &&
3193 (MemSem & SPIRV::MemorySemantics::ImageMemory) > 0)) &&
3194 "Device Scope must set UniformMemory and ImageMemory semantic "
3195 "in Barrier instruction");
3196
3197 Register MemSemReg = buildI32Constant(MemSem, I);
3198 Register ScopeReg = buildI32Constant(Scope, I);
3199 MachineBasicBlock &BB = *I.getParent();
3200 auto MI =
3201 BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType)).addUse(ScopeReg);
3202
3203 // OpControlBarrier needs to also set Execution Scope
3204 if (WithGroupSync) {
3205 MI.addUse(ScopeReg);
3206 }
3207
3208 MI.addUse(MemSemReg).constrainAllUses(TII, TRI, RBI);
3209 return true;
3210}
3211
3212bool SPIRVInstructionSelector::selectWaveActiveCountBits(
3213 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3214
3215 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3216 SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
3217 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType));
3218 if (!selectWaveOpInst(BallotReg, BallotType, I,
3219 SPIRV::OpGroupNonUniformBallot))
3220 return false;
3221
3222 MachineBasicBlock &BB = *I.getParent();
3223 BuildMI(BB, I, I.getDebugLoc(),
3224 TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3225 .addDef(ResVReg)
3226 .addUse(GR.getSPIRVTypeID(ResType))
3227 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3228 !STI.isShader()))
3229 .addImm(SPIRV::GroupOperation::Reduce)
3230 .addUse(BallotReg)
3231 .constrainAllUses(TII, TRI, RBI);
3232
3233 return true;
3234}
3235
3236unsigned getVectorSizeOrOne(SPIRVTypeInst Type) {
3237
3238 if (Type->getOpcode() != SPIRV::OpTypeVector)
3239 return 1;
3240
3241 // Operand(2) is the vector size
3242 return Type->getOperand(2).getImm();
3243}
3244
3245bool SPIRVInstructionSelector::selectWaveActiveAllEqual(Register ResVReg,
3246 SPIRVTypeInst ResType,
3247 MachineInstr &I) const {
3248 MachineBasicBlock &BB = *I.getParent();
3249 const DebugLoc &DL = I.getDebugLoc();
3250
3251 // Input to the intrinsic
3252 Register InputReg = I.getOperand(2).getReg();
3253 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputReg);
3254
3255 // Determine if input is vector
3256 unsigned NumElems = getVectorSizeOrOne(InputType);
3257 bool IsVector = NumElems > 1;
3258
3259 // Determine element types
3260 SPIRVTypeInst ElemInputType = InputType;
3261 SPIRVTypeInst ElemBoolType = ResType;
3262 if (IsVector) {
3263 ElemInputType = GR.getSPIRVTypeForVReg(InputType->getOperand(1).getReg());
3264 ElemBoolType = GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg());
3265 }
3266
3267 // Subgroup scope constant
3268 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3269 Register ScopeConst = GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy,
3270 TII, !STI.isShader());
3271
3272 // Scalar case
3273 if (!IsVector) {
3274 return selectWaveOpInst(ResVReg, ElemBoolType, I,
3275 SPIRV::OpGroupNonUniformAllEqual);
3276 }
3277
3278 // Vector case
3279 SmallVector<Register, 4> ElementResults;
3280 ElementResults.reserve(NumElems);
3281
3282 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
3283 // Extract element
3284 Register ElemInput = InputReg;
3285 Register Extracted =
3286 MRI->createVirtualRegister(GR.getRegClass(ElemInputType));
3287
3288 BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeExtract))
3289 .addDef(Extracted)
3290 .addUse(GR.getSPIRVTypeID(ElemInputType))
3291 .addUse(InputReg)
3292 .addImm(Idx)
3293 .constrainAllUses(TII, TRI, RBI);
3294
3295 ElemInput = Extracted;
3296
3297 // Emit per-element AllEqual
3298 Register ElemResult =
3299 MRI->createVirtualRegister(GR.getRegClass(ElemBoolType));
3300
3301 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformAllEqual))
3302 .addDef(ElemResult)
3303 .addUse(GR.getSPIRVTypeID(ElemBoolType))
3304 .addUse(ScopeConst)
3305 .addUse(ElemInput)
3306 .constrainAllUses(TII, TRI, RBI);
3307
3308 ElementResults.push_back(ElemResult);
3309 }
3310
3311 // Reconstruct vector<bool>
3312 auto MIB = BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeConstruct))
3313 .addDef(ResVReg)
3314 .addUse(GR.getSPIRVTypeID(ResType));
3315 for (Register R : ElementResults)
3316 MIB.addUse(R);
3317
3318 MIB.constrainAllUses(TII, TRI, RBI);
3319
3320 return true;
3321}
3322
3323bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg,
3324 SPIRVTypeInst ResType,
3325 MachineInstr &I) const {
3326
3327 assert(I.getNumOperands() == 3);
3328
3329 auto Op = I.getOperand(2);
3330 assert(Op.isReg());
3331
3332 MachineBasicBlock &BB = *I.getParent();
3333 DebugLoc DL = I.getDebugLoc();
3334
3335 Register InputRegister = Op.getReg();
3336 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3337
3338 if (!InputType)
3339 report_fatal_error("Input Type could not be determined.");
3340
3341 if (InputType->getOpcode() != SPIRV::OpTypeBool)
3342 report_fatal_error("WavePrefixBitCount requires boolean input");
3343
3344 // Types
3345 SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3346
3347 // Ballot result type: vector<uint32>
3348 // Match DXC: %v4uint for Subgroup size
3349 SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(Int32Ty, 4, I, TII);
3350
3351 // Create a vreg for the ballot result
3352 Register BallotVReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3353
3354 // 1. OpGroupNonUniformBallot
3355 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallot))
3356 .addDef(BallotVReg)
3357 .addUse(GR.getSPIRVTypeID(BallotTy))
3358 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3359 .addUse(InputRegister)
3360 .constrainAllUses(TII, TRI, RBI);
3361
3362 // 2. OpGroupNonUniformBallotBitCount
3363 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3364 .addDef(ResVReg)
3365 .addUse(GR.getSPIRVTypeID(ResType))
3366 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3367 .addImm(SPIRV::GroupOperation::ExclusiveScan)
3368 .addUse(BallotVReg)
3369 .constrainAllUses(TII, TRI, RBI);
3370
3371 return true;
3372}
3373
3374bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
3375 SPIRVTypeInst ResType,
3376 MachineInstr &I,
3377 bool IsUnsigned) const {
3378 return selectWaveReduce(
3379 ResVReg, ResType, I, IsUnsigned,
3380 [&](Register InputRegister, bool IsUnsigned) {
3381 const bool IsFloatTy =
3382 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3383 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
3384 : SPIRV::OpGroupNonUniformSMax;
3385 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
3386 });
3387}
3388
3389bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
3390 SPIRVTypeInst ResType,
3391 MachineInstr &I,
3392 bool IsUnsigned) const {
3393 return selectWaveReduce(
3394 ResVReg, ResType, I, IsUnsigned,
3395 [&](Register InputRegister, bool IsUnsigned) {
3396 const bool IsFloatTy =
3397 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3398 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
3399 : SPIRV::OpGroupNonUniformSMin;
3400 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
3401 });
3402}
3403
3404bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
3405 SPIRVTypeInst ResType,
3406 MachineInstr &I) const {
3407 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3408 [&](Register InputRegister, bool IsUnsigned) {
3409 bool IsFloatTy = GR.isScalarOrVectorOfType(
3410 InputRegister, SPIRV::OpTypeFloat);
3411 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
3412 : SPIRV::OpGroupNonUniformIAdd;
3413 });
3414}
3415
3416bool SPIRVInstructionSelector::selectWaveReduceProduct(Register ResVReg,
3417 SPIRVTypeInst ResType,
3418 MachineInstr &I) const {
3419 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3420 [&](Register InputRegister, bool IsUnsigned) {
3421 bool IsFloatTy = GR.isScalarOrVectorOfType(
3422 InputRegister, SPIRV::OpTypeFloat);
3423 return IsFloatTy ? SPIRV::OpGroupNonUniformFMul
3424 : SPIRV::OpGroupNonUniformIMul;
3425 });
3426}
3427
3428template <typename PickOpcodeFn>
3429bool SPIRVInstructionSelector::selectWaveReduce(
3430 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
3431 PickOpcodeFn &&PickOpcode) const {
3432 assert(I.getNumOperands() == 3);
3433 assert(I.getOperand(2).isReg());
3434 MachineBasicBlock &BB = *I.getParent();
3435 Register InputRegister = I.getOperand(2).getReg();
3436 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3437
3438 if (!InputType)
3439 report_fatal_error("Input Type could not be determined.");
3440
3441 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3442 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
3443 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3444 .addDef(ResVReg)
3445 .addUse(GR.getSPIRVTypeID(ResType))
3446 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3447 !STI.isShader()))
3448 .addImm(SPIRV::GroupOperation::Reduce)
3449 .addUse(I.getOperand(2).getReg())
3450 .constrainAllUses(TII, TRI, RBI);
3451 return true;
3452}
3453
3454bool SPIRVInstructionSelector::selectWaveReduceOp(Register ResVReg,
3455 SPIRVTypeInst ResType,
3456 MachineInstr &I,
3457 unsigned Opcode) const {
3458 return selectWaveReduce(
3459 ResVReg, ResType, I, false,
3460 [&](Register InputRegister, bool IsUnsigned) { return Opcode; });
3461}
3462
3463bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
3464 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3465 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3466 [&](Register InputRegister, bool IsUnsigned) {
3467 bool IsFloatTy = GR.isScalarOrVectorOfType(
3468 InputRegister, SPIRV::OpTypeFloat);
3469 return IsFloatTy
3470 ? SPIRV::OpGroupNonUniformFAdd
3471 : SPIRV::OpGroupNonUniformIAdd;
3472 });
3473}
3474
3475bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
3476 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3477 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3478 [&](Register InputRegister, bool IsUnsigned) {
3479 bool IsFloatTy = GR.isScalarOrVectorOfType(
3480 InputRegister, SPIRV::OpTypeFloat);
3481 return IsFloatTy
3482 ? SPIRV::OpGroupNonUniformFMul
3483 : SPIRV::OpGroupNonUniformIMul;
3484 });
3485}
3486
3487template <typename PickOpcodeFn>
3488bool SPIRVInstructionSelector::selectWaveExclusiveScan(
3489 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
3490 PickOpcodeFn &&PickOpcode) const {
3491 assert(I.getNumOperands() == 3);
3492 assert(I.getOperand(2).isReg());
3493 MachineBasicBlock &BB = *I.getParent();
3494 Register InputRegister = I.getOperand(2).getReg();
3495 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3496
3497 if (!InputType)
3498 report_fatal_error("Input Type could not be determined.");
3499
3500 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3501 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
3502 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3503 .addDef(ResVReg)
3504 .addUse(GR.getSPIRVTypeID(ResType))
3505 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3506 !STI.isShader()))
3507 .addImm(SPIRV::GroupOperation::ExclusiveScan)
3508 .addUse(I.getOperand(2).getReg())
3509 .constrainAllUses(TII, TRI, RBI);
3510 return true;
3511}
3512
3513bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg,
3514 SPIRVTypeInst ResType,
3515 MachineInstr &I,
3516 unsigned Direction) const {
3517 assert(I.getNumOperands() == 3);
3518 assert(I.getOperand(2).isReg());
3519 MachineBasicBlock &BB = *I.getParent();
3520 Register InputRegister = I.getOperand(2).getReg();
3521
3522 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3523 bool ZeroAsNull = !STI.isShader();
3524 Register DirectionReg =
3525 GR.getOrCreateConstInt(Direction, I, IntTy, TII, ZeroAsNull);
3526 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformQuadSwap))
3527 .addDef(ResVReg)
3528 .addUse(GR.getSPIRVTypeID(ResType))
3529 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3530 ZeroAsNull))
3531 .addUse(InputRegister)
3532 .addUse(DirectionReg)
3533 .constrainAllUses(TII, TRI, RBI);
3534 return true;
3535}
3536
3537bool SPIRVInstructionSelector::selectBitreverse16(Register ResVReg,
3538 SPIRVTypeInst ResType,
3539 MachineInstr &I,
3540 Register Op) const {
3541 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3542 Register ShiftConst = GR.getOrCreateConstInt(16, I, Int32Type, TII);
3543 unsigned ShiftOp = SPIRV::OpShiftRightLogicalS;
3544
3545 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
3546 const unsigned ExtendOpcode = GR.isScalarOrVectorSigned(ResType)
3547 ? SPIRV::OpSConvert
3548 : SPIRV::OpUConvert;
3549
3550 if (N > 1) {
3551 Int32Type = GR.getOrCreateSPIRVVectorType(Int32Type, N, I, TII);
3552 ShiftOp = SPIRV::OpShiftRightLogicalV;
3553
3554 // Vector shifts require a composite constant
3555 const Register CompositeReg =
3556 MRI->createVirtualRegister(GR.getRegClass(Int32Type));
3557 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3558 TII.get(SPIRV::OpConstantComposite))
3559 .addDef(CompositeReg)
3560 .addUse(GR.getSPIRVTypeID(Int32Type));
3561 for (unsigned It = 0; It < N; ++It)
3562 MIB.addUse(ShiftConst);
3563 MIB.constrainAllUses(TII, TRI, RBI);
3564
3565 ShiftConst = CompositeReg;
3566 }
3567
3568 // Converts the i16 input to i32 (or vector of i32)
3569 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
3570 if (!selectOpWithSrcs(ExtReg, Int32Type, I, {Op}, ExtendOpcode))
3571 return false;
3572
3573 // Perform bitreverse on the i32 value
3574 Register BitrevReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
3575 if (!selectBitreverseNative(BitrevReg, Int32Type, I, ExtReg))
3576 return false;
3577
3578 // Shift the bit-reversed value to get the final result.
3579 Register ShiftReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
3580 if (!selectOpWithSrcs(ShiftReg, Int32Type, I, {BitrevReg, ShiftConst},
3581 ShiftOp))
3582 return false;
3583
3584 // Finally, convert the result back to i16 (or vector of i16).
3585 return selectOpWithSrcs(ResVReg, ResType, I, {ShiftReg}, ExtendOpcode);
3586}
3587
3588bool SPIRVInstructionSelector::selectBitreverseNative(Register ResVReg,
3589 SPIRVTypeInst ResType,
3590 MachineInstr &I,
3591 Register Op) const {
3592 MachineBasicBlock &BB = *I.getParent();
3593 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
3594 .addDef(ResVReg)
3595 .addUse(GR.getSPIRVTypeID(ResType))
3596 .addUse(Op)
3597 .constrainAllUses(TII, TRI, RBI);
3598 return true;
3599}
3600
3601bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
3602 SPIRVTypeInst ResType,
3603 MachineInstr &I) const {
3604 Register OpReg = I.getOperand(1).getReg();
3605
3606 // TODO: Fix shader behavior in case of VK_KHR_maintenance9 extension is set
3607 if (STI.isShader()) {
3608 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
3609 switch (GR.getScalarOrVectorBitWidth(OpType)) {
3610 case 16:
3611 return selectBitreverse16(ResVReg, ResType, I, OpReg);
3612 default:
3613 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
3614 }
3615 }
3616
3617 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions))
3618 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
3619
3620 // Expansion bitreverse using bit manipulation operations
3621 // Algo: https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
3622 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
3623 // TODO: add support for any bit width and bitwidth more than 64.
3624 if (BitWidth > 64 || !isPowerOf2_32(BitWidth))
3625 return false;
3626
3627 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
3628
3629 unsigned AndOp = SPIRV::OpBitwiseAndS;
3630 unsigned OrOp = SPIRV::OpBitwiseOrS;
3631 unsigned ShlOp = SPIRV::OpShiftLeftLogicalS;
3632 unsigned ShrOp = SPIRV::OpShiftRightLogicalS;
3633 if (N > 1) {
3634 AndOp = SPIRV::OpBitwiseAndV;
3635 OrOp = SPIRV::OpBitwiseOrV;
3636 ShlOp = SPIRV::OpShiftLeftLogicalV;
3637 ShrOp = SPIRV::OpShiftRightLogicalV;
3638 }
3639
3640 // Helper, one swap per step: ((input>>shift)&mask)|((input&mask)<<shift),
3641 // RPN: input shift >> mask & input mask & shift << |
3642 auto SwapBits = [&](const Register Input, const uint64_t Mask,
3643 const unsigned Shift) -> Register {
3644 auto CreateConst = [&](const uint64_t Value) -> Register {
3645 if (N == 1)
3646 return GR.getOrCreateConstInt(
3647 Value, I, GR.retrieveScalarOrVectorIntType(ResType), TII);
3648 return GR.getOrCreateConstVector(Value, I, ResType, TII);
3649 };
3650
3651 Register MaskReg = CreateConst(Mask);
3652 Register ShiftReg = CreateConst(Shift);
3653 Register T1 = MRI->createVirtualRegister(GR.getRegClass(ResType));
3654 Register T2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
3655 Register T3 = MRI->createVirtualRegister(GR.getRegClass(ResType));
3656 Register T4 = MRI->createVirtualRegister(GR.getRegClass(ResType));
3657 Register Result = MRI->createVirtualRegister(GR.getRegClass(ResType));
3658
3659 if (!selectOpWithSrcs(T1, ResType, I, {Input, ShiftReg}, ShrOp) ||
3660 !selectOpWithSrcs(T2, ResType, I, {T1, MaskReg}, AndOp) ||
3661 !selectOpWithSrcs(T3, ResType, I, {Input, MaskReg}, AndOp) ||
3662 !selectOpWithSrcs(T4, ResType, I, {T3, ShiftReg}, ShlOp) ||
3663 !selectOpWithSrcs(Result, ResType, I, {T2, T4}, OrOp))
3664 return Register();
3665
3666 return Result;
3667 };
3668
3669 unsigned Shift = BitWidth;
3670 Register Result = OpReg;
3671 uint64_t Mask = ~0ull;
3672 while ((Shift >>= 1) > 0) {
3673 Mask ^= (Mask << Shift);
3674 Result = SwapBits(Result, Mask, Shift);
3675 if (!Result.isValid())
3676 return false;
3677 }
3678
3679 return BuildCOPY(ResVReg, Result, I);
3680}
3681
3682bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
3683 SPIRVTypeInst ResType,
3684 MachineInstr &I) const {
3685 // There is no way to implement `freeze` correctly without support on SPIR-V
3686 // standard side, but we may at least address a simple (static) case when
3687 // undef/poison value presence is obvious. The main benefit of even
3688 // incomplete `freeze` support is preventing of translation from crashing due
3689 // to lack of support on legalization and instruction selection steps.
3690 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg())
3691 return false;
3692 Register OpReg = I.getOperand(1).getReg();
3693 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
3694 if (Def->getOpcode() == TargetOpcode::COPY)
3695 Def = MRI->getVRegDef(Def->getOperand(1).getReg());
3696 Register Reg;
3697 switch (Def->getOpcode()) {
3698 case SPIRV::ASSIGN_TYPE:
3699 if (MachineInstr *AssignToDef =
3700 MRI->getVRegDef(Def->getOperand(1).getReg())) {
3701 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
3702 Reg = Def->getOperand(2).getReg();
3703 }
3704 break;
3705 case SPIRV::OpUndef:
3706 Reg = Def->getOperand(1).getReg();
3707 break;
3708 }
3709 unsigned DestOpCode;
3710 if (Reg.isValid()) {
3711 DestOpCode = SPIRV::OpConstantNull;
3712 } else {
3713 DestOpCode = TargetOpcode::COPY;
3714 Reg = OpReg;
3715 }
3716 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
3717 .addDef(I.getOperand(0).getReg())
3718 .addUse(Reg)
3719 .constrainAllUses(TII, TRI, RBI);
3720 return true;
3721 }
3722 return false;
3723}
3724
3725bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
3726 SPIRVTypeInst ResType,
3727 MachineInstr &I) const {
3728 unsigned N = 0;
3729 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3730 N = GR.getScalarOrVectorComponentCount(ResType);
3731 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3732 N = getArrayComponentCount(MRI, ResType);
3733 else
3734 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result");
3735 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
3736 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent");
3737
3738 // check if we may construct a constant vector
3739 bool IsConst = true;
3740 for (unsigned i = I.getNumExplicitDefs();
3741 i < I.getNumExplicitOperands() && IsConst; ++i)
3742 if (!isConstReg(MRI, I.getOperand(i).getReg()))
3743 IsConst = false;
3744
3745 if (!IsConst && N < 2)
3746 report_fatal_error(
3747 "There must be at least two constituent operands in a vector");
3748
3749 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
3750 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3751 TII.get(IsConst ? SPIRV::OpConstantComposite
3752 : SPIRV::OpCompositeConstruct))
3753 .addDef(ResVReg)
3754 .addUse(GR.getSPIRVTypeID(ResType));
3755 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
3756 MIB.addUse(I.getOperand(i).getReg());
3757 MIB.constrainAllUses(TII, TRI, RBI);
3758 return true;
3759}
3760
3761bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
3762 SPIRVTypeInst ResType,
3763 MachineInstr &I) const {
3764 unsigned N = 0;
3765 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3766 N = GR.getScalarOrVectorComponentCount(ResType);
3767 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3768 N = getArrayComponentCount(MRI, ResType);
3769 else
3770 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
3771
3772 unsigned OpIdx = I.getNumExplicitDefs();
3773 if (!I.getOperand(OpIdx).isReg())
3774 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
3775
3776 // check if we may construct a constant vector
3777 Register OpReg = I.getOperand(OpIdx).getReg();
3778 bool IsConst = isConstReg(MRI, OpReg);
3779
3780 if (!IsConst && N < 2)
3781 report_fatal_error(
3782 "There must be at least two constituent operands in a vector");
3783
3784 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
3785 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3786 TII.get(IsConst ? SPIRV::OpConstantComposite
3787 : SPIRV::OpCompositeConstruct))
3788 .addDef(ResVReg)
3789 .addUse(GR.getSPIRVTypeID(ResType));
3790 for (unsigned i = 0; i < N; ++i)
3791 MIB.addUse(OpReg);
3792 MIB.constrainAllUses(TII, TRI, RBI);
3793 return true;
3794}
3795
3796bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
3797 SPIRVTypeInst ResType,
3798 MachineInstr &I) const {
3799
3800 unsigned Opcode;
3801
3802 if (STI.canUseExtension(
3803 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
3804 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) {
3805 Opcode = SPIRV::OpDemoteToHelperInvocation;
3806 } else {
3807 Opcode = SPIRV::OpKill;
3808 // OpKill must be the last operation of any basic block.
3809 if (MachineInstr *NextI = I.getNextNode()) {
3810 GR.invalidateMachineInstr(NextI);
3811 NextI->eraseFromParent();
3812 }
3813 }
3814
3815 MachineBasicBlock &BB = *I.getParent();
3816 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3817 .constrainAllUses(TII, TRI, RBI);
3818 return true;
3819}
3820
3821bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
3822 SPIRVTypeInst ResType, unsigned CmpOpc,
3823 MachineInstr &I) const {
3824 Register Cmp0 = I.getOperand(2).getReg();
3825 Register Cmp1 = I.getOperand(3).getReg();
3826 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
3827 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
3828 "CMP operands should have the same type");
3829 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
3830 .addDef(ResVReg)
3831 .addUse(GR.getSPIRVTypeID(ResType))
3832 .addUse(Cmp0)
3833 .addUse(Cmp1)
3834 .setMIFlags(I.getFlags())
3835 .constrainAllUses(TII, TRI, RBI);
3836 return true;
3837}
3838
3839bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
3840 SPIRVTypeInst ResType,
3841 MachineInstr &I) const {
3842 auto Pred = I.getOperand(1).getPredicate();
3843 unsigned CmpOpc;
3844
3845 Register CmpOperand = I.getOperand(2).getReg();
3846 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
3847 CmpOpc = getPtrCmpOpcode(Pred);
3848 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
3849 CmpOpc = getBoolCmpOpcode(Pred);
3850 else
3851 CmpOpc = getICmpOpcode(Pred);
3852 return selectCmp(ResVReg, ResType, CmpOpc, I);
3853}
3854
3855Register
3856SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
3857 SPIRVTypeInst ResType) const {
3858 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
3859 SPIRVTypeInst SpvI32Ty =
3860 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
3861 // Find a constant in DT or build a new one.
3862 auto ConstInt = ConstantInt::get(LLVMTy, Val);
3863 Register NewReg = GR.find(ConstInt, GR.CurMF);
3864 if (!NewReg.isValid()) {
3865 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
3866 MachineBasicBlock &BB = *I.getParent();
3867 MachineInstr *MI =
3868 Val == 0
3869 ? BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
3870 .addDef(NewReg)
3871 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
3872 : BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
3873 .addDef(NewReg)
3874 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
3875 .addImm(APInt(32, Val).getZExtValue());
3876 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
3877 GR.add(ConstInt, MI);
3878 }
3879 return NewReg;
3880}
3881
3882bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
3883 SPIRVTypeInst ResType,
3884 MachineInstr &I) const {
3885 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
3886 return selectCmp(ResVReg, ResType, CmpOp, I);
3887}
3888
3889bool SPIRVInstructionSelector::selectExp10(Register ResVReg,
3890 SPIRVTypeInst ResType,
3891 MachineInstr &I) const {
3892 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
3893 return selectExtInst(ResVReg, ResType, I, CL::exp10);
3894 }
3895
3896 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
3897 /// There is no exp10 in GLSL. Use exp10(x) = exp2(x * log2(10)) instead
3898 /// log2(10) ~= 3.3219280948874l
3899
3900 if (ResType->getOpcode() != SPIRV::OpTypeVector &&
3901 ResType->getOpcode() != SPIRV::OpTypeFloat)
3902 return false;
3903
3904 MachineIRBuilder MIRBuilder(I);
3905
3906 SPIRVTypeInst SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector
3907 ? SPIRVTypeInst(GR.getSPIRVTypeForVReg(
3908 ResType->getOperand(1).getReg()))
3909 : ResType;
3910
3911 assert(SpirvScalarType->getOperand(1).getImm() == 32 &&
3912 "only float operands supported by GLSL extended math");
3913
3914 Register ConstReg = GR.buildConstantFP(APFloat(3.3219280948874f),
3915 MIRBuilder, SpirvScalarType);
3916 Register ArgReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3917 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
3918 ? SPIRV::OpVectorTimesScalar
3919 : SPIRV::OpFMulS;
3920
3921 if (!selectOpWithSrcs(ArgReg, ResType, I,
3922 {I.getOperand(1).getReg(), ConstReg}, Opcode))
3923 return false;
3924 if (!selectExtInst(ResVReg, ResType, I,
3925 {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}}, false,
3926 false, {ArgReg}))
3927 return false;
3928
3929 return true;
3930 }
3931
3932 return false;
3933}
3934
3935Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
3936 MachineInstr &I) const {
3937 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3938 bool ZeroAsNull = !STI.isShader();
3939 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3940 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
3941 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
3942}
3943
3944bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
3945 Register Reg) const {
3946 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(Reg);
3947 if (!Type)
3948 return false;
3949 SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type);
3950 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
3951 return false;
3952
3953 auto IsZero = [this](Register Reg) {
3954 MachineInstr *Def = getDefInstrMaybeConstant(Reg, MRI);
3955 if (!Def)
3956 return false;
3957
3958 if (Def->getOpcode() == SPIRV::OpConstantNull)
3959 return true;
3960
3961 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
3962 Def->getOpcode() == SPIRV::OpConstantI)
3963 return getIConstVal(Reg, MRI) == 0;
3964
3965 return false;
3966 };
3967
3968 if (IsZero(Reg))
3969 return true;
3970
3971 MachineInstr *Def = MRI->getVRegDef(Reg);
3972 if (!Def)
3973 return false;
3974
3975 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
3976 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
3977 cast<GIntrinsic>(Def)->getIntrinsicID() ==
3978 Intrinsic::spv_const_composite)) {
3979 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
3980 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
3981 if (!IsZero(Def->getOperand(i).getReg()))
3982 return false;
3983 }
3984 return true;
3985 }
3986
3987 return false;
3988}
3989
3990Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
3991 MachineInstr &I) const {
3992 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3993 bool ZeroAsNull = !STI.isShader();
3994 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
3995 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3996 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
3997 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
3998}
3999
4000Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
4001 MachineInstr &I) const {
4002 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4003 bool ZeroAsNull = !STI.isShader();
4004 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType));
4005 if (ResType->getOpcode() == SPIRV::OpTypeVector)
4006 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull);
4007 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull);
4008}
4009
4010Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
4011 SPIRVTypeInst ResType,
4012 MachineInstr &I) const {
4013 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4014 APInt One =
4015 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
4016 if (ResType->getOpcode() == SPIRV::OpTypeVector)
4017 return GR.getOrCreateConstVector(One, I, ResType, TII);
4018 return GR.getOrCreateConstInt(One, I, ResType, TII);
4019}
4020
4021bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
4022 SPIRVTypeInst ResType,
4023 MachineInstr &I) const {
4024 Register SelectFirstArg = I.getOperand(2).getReg();
4025 Register SelectSecondArg = I.getOperand(3).getReg();
4026 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
4027 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
4028
4029 bool IsFloatTy =
4030 GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypeFloat);
4031 bool IsPtrTy =
4032 GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypePointer);
4033 bool IsVectorTy = GR.getSPIRVTypeForVReg(SelectFirstArg)->getOpcode() ==
4034 SPIRV::OpTypeVector;
4035
4036 bool IsScalarBool =
4037 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
4038 unsigned Opcode;
4039 if (IsVectorTy) {
4040 if (IsFloatTy) {
4041 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
4042 } else if (IsPtrTy) {
4043 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
4044 } else {
4045 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
4046 }
4047 } else {
4048 if (IsFloatTy) {
4049 Opcode = IsScalarBool ? SPIRV::OpSelectSFSCond : SPIRV::OpSelectVFVCond;
4050 } else if (IsPtrTy) {
4051 Opcode = IsScalarBool ? SPIRV::OpSelectSPSCond : SPIRV::OpSelectVPVCond;
4052 } else {
4053 Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
4054 }
4055 }
4056 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
4057 .addDef(ResVReg)
4058 .addUse(GR.getSPIRVTypeID(ResType))
4059 .addUse(I.getOperand(1).getReg())
4060 .addUse(SelectFirstArg)
4061 .addUse(SelectSecondArg)
4062 .constrainAllUses(TII, TRI, RBI);
4063 return true;
4064}
4065
4066// This function is used to extend a bool or a vector of bools into an integer
4067// or vector of integers.
4068bool SPIRVInstructionSelector::selectBoolToInt(Register ResVReg,
4069 SPIRVTypeInst ResType,
4070 Register BooleanVReg,
4071 MachineInstr &InsertAt,
4072 bool IsSigned) const {
4073 // To extend a bool, we need to use OpSelect between constants.
4074 Register ZeroReg = buildZerosVal(ResType, InsertAt);
4075 Register OneReg = buildOnesVal(IsSigned, ResType, InsertAt);
4076 bool IsScalarBool = GR.isScalarOfType(BooleanVReg, SPIRV::OpTypeBool);
4077 unsigned Opcode =
4078 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
4079 BuildMI(*InsertAt.getParent(), InsertAt, InsertAt.getDebugLoc(),
4080 TII.get(Opcode))
4081 .addDef(ResVReg)
4082 .addUse(GR.getSPIRVTypeID(ResType))
4083 .addUse(BooleanVReg)
4084 .addUse(OneReg)
4085 .addUse(ZeroReg)
4086 .constrainAllUses(TII, TRI, RBI);
4087 return true;
4088}
4089
4090bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
4091 SPIRVTypeInst ResType,
4092 MachineInstr &I, bool IsSigned,
4093 unsigned Opcode) const {
4094 Register SrcReg = I.getOperand(1).getReg();
4095 // We can convert bool value directly to float type without OpConvert*ToF,
4096 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
4097 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
4098 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4099 SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
4100 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
4101 const unsigned NumElts = ResType->getOperand(2).getImm();
4102 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
4103 }
4104 SrcReg = createVirtualRegister(TmpType, &GR, MRI, MRI->getMF());
4105 selectBoolToInt(SrcReg, TmpType, I.getOperand(1).getReg(), I, false);
4106 }
4107 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
4108}
4109
4110bool SPIRVInstructionSelector::selectExt(Register ResVReg,
4111 SPIRVTypeInst ResType, MachineInstr &I,
4112 bool IsSigned) const {
4113 Register SrcReg = I.getOperand(1).getReg();
4114 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
4115 return selectBoolToInt(ResVReg, ResType, I.getOperand(1).getReg(), I,
4116 IsSigned);
4117
4118 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
4119 if (ResType == SrcType)
4120 return BuildCOPY(ResVReg, SrcReg, I);
4121
4122 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
4123 return selectUnOp(ResVReg, ResType, I, Opcode);
4124}
4125
4126bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
4127 SPIRVTypeInst ResType,
4128 MachineInstr &I,
4129 bool IsSigned) const {
4130 MachineIRBuilder MIRBuilder(I);
4131 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4132 MachineBasicBlock &BB = *I.getParent();
4133 // Ensure we have bool.
4134 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
4135 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4136 if (N > 1)
4137 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
4138 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
4139 // Build less-than-equal and less-than.
4140 // TODO: replace with one-liner createVirtualRegister() from
4141 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
4142 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
4143 MRI->setType(IsLessEqReg, LLT::scalar(64));
4144 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF());
4145 BuildMI(BB, I, I.getDebugLoc(),
4146 TII.get(IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
4147 .addDef(IsLessEqReg)
4148 .addUse(BoolTypeReg)
4149 .addUse(I.getOperand(1).getReg())
4150 .addUse(I.getOperand(2).getReg())
4151 .constrainAllUses(TII, TRI, RBI);
4152 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
4153 MRI->setType(IsLessReg, LLT::scalar(64));
4154 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF());
4155 BuildMI(BB, I, I.getDebugLoc(),
4156 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
4157 .addDef(IsLessReg)
4158 .addUse(BoolTypeReg)
4159 .addUse(I.getOperand(1).getReg())
4160 .addUse(I.getOperand(2).getReg())
4161 .constrainAllUses(TII, TRI, RBI);
4162 // Build selects.
4163 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
4164 Register NegOneOrZeroReg =
4165 MRI->createVirtualRegister(GR.getRegClass(ResType));
4166 MRI->setType(NegOneOrZeroReg, LLT::scalar(64));
4167 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF());
4168 unsigned SelectOpcode =
4169 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
4170 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
4171 .addDef(NegOneOrZeroReg)
4172 .addUse(ResTypeReg)
4173 .addUse(IsLessReg)
4174 .addUse(buildOnesVal(true, ResType, I)) // -1
4175 .addUse(buildZerosVal(ResType, I))
4176 .constrainAllUses(TII, TRI, RBI);
4177 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
4178 .addDef(ResVReg)
4179 .addUse(ResTypeReg)
4180 .addUse(IsLessEqReg)
4181 .addUse(NegOneOrZeroReg) // -1 or 0
4182 .addUse(buildOnesVal(false, ResType, I))
4183 .constrainAllUses(TII, TRI, RBI);
4184 return true;
4185}
4186
4187bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
4188 Register ResVReg,
4189 MachineInstr &I,
4190 SPIRVTypeInst IntTy,
4191 SPIRVTypeInst BoolTy) const {
4192 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
4193 Register BitIntReg = createVirtualRegister(IntTy, &GR, MRI, MRI->getMF());
4194 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
4195 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
4196 Register Zero = buildZerosVal(IntTy, I);
4197 Register One = buildOnesVal(false, IntTy, I);
4198 MachineBasicBlock &BB = *I.getParent();
4199 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4200 .addDef(BitIntReg)
4201 .addUse(GR.getSPIRVTypeID(IntTy))
4202 .addUse(IntReg)
4203 .addUse(One)
4204 .constrainAllUses(TII, TRI, RBI);
4205 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
4206 .addDef(ResVReg)
4207 .addUse(GR.getSPIRVTypeID(BoolTy))
4208 .addUse(BitIntReg)
4209 .addUse(Zero)
4210 .constrainAllUses(TII, TRI, RBI);
4211 return true;
4212}
4213
4214bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
4215 SPIRVTypeInst ResType,
4216 MachineInstr &I) const {
4217 Register IntReg = I.getOperand(1).getReg();
4218 const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(IntReg);
4219 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
4220 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
4221 if (ArgType == ResType)
4222 return BuildCOPY(ResVReg, IntReg, I);
4223 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
4224 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
4225 return selectUnOp(ResVReg, ResType, I, Opcode);
4226}
4227
4228bool SPIRVInstructionSelector::selectConst(Register ResVReg,
4229 SPIRVTypeInst ResType,
4230 MachineInstr &I) const {
4231 unsigned Opcode = I.getOpcode();
4232 unsigned TpOpcode = ResType->getOpcode();
4233 Register Reg;
4234 if (TpOpcode == SPIRV::OpTypePointer || TpOpcode == SPIRV::OpTypeEvent) {
4235 assert(Opcode == TargetOpcode::G_CONSTANT &&
4236 I.getOperand(1).getCImm()->isZero());
4237 MachineBasicBlock &DepMBB = I.getMF()->front();
4238 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
4239 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, ResType);
4240 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
4241 Reg = GR.getOrCreateConstFP(I.getOperand(1).getFPImm()->getValue(), I,
4242 ResType, TII, !STI.isShader());
4243 } else {
4244 Reg = GR.getOrCreateConstInt(I.getOperand(1).getCImm()->getValue(), I,
4245 ResType, TII, !STI.isShader());
4246 }
4247 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
4248}
4249
4250bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
4251 SPIRVTypeInst ResType,
4252 MachineInstr &I) const {
4253 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
4254 .addDef(ResVReg)
4255 .addUse(GR.getSPIRVTypeID(ResType))
4256 .constrainAllUses(TII, TRI, RBI);
4257 return true;
4258}
4259
4260bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
4261 SPIRVTypeInst ResType,
4262 MachineInstr &I) const {
4263 MachineBasicBlock &BB = *I.getParent();
4264 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
4265 .addDef(ResVReg)
4266 .addUse(GR.getSPIRVTypeID(ResType))
4267 // object to insert
4268 .addUse(I.getOperand(3).getReg())
4269 // composite to insert into
4270 .addUse(I.getOperand(2).getReg());
4271 for (unsigned i = 4; i < I.getNumOperands(); i++)
4272 MIB.addImm(foldImm(I.getOperand(i), MRI));
4273 MIB.constrainAllUses(TII, TRI, RBI);
4274 return true;
4275}
4276
4277bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
4278 SPIRVTypeInst ResType,
4279 MachineInstr &I) const {
4280 Type *MaybeResTy = nullptr;
4281 StringRef ResName;
4282 if (GR.findValueAttrs(&I, MaybeResTy, ResName) &&
4283 MaybeResTy != GR.getTypeForSPIRVType(ResType)) {
4284 assert((!MaybeResTy || MaybeResTy->isAggregateType()) &&
4285 "Expected aggregate type for extractv instruction");
4286 ResType = GR.getOrCreateSPIRVType(MaybeResTy, I,
4287 SPIRV::AccessQualifier::ReadWrite, false);
4288 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *I.getMF());
4289 }
4290 MachineBasicBlock &BB = *I.getParent();
4291 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
4292 .addDef(ResVReg)
4293 .addUse(GR.getSPIRVTypeID(ResType))
4294 .addUse(I.getOperand(2).getReg());
4295 for (unsigned i = 3; i < I.getNumOperands(); i++)
4296 MIB.addImm(foldImm(I.getOperand(i), MRI));
4297 MIB.constrainAllUses(TII, TRI, RBI);
4298 return true;
4299}
4300
4301bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
4302 SPIRVTypeInst ResType,
4303 MachineInstr &I) const {
4304 if (getImm(I.getOperand(4), MRI))
4305 return selectInsertVal(ResVReg, ResType, I);
4306 MachineBasicBlock &BB = *I.getParent();
4307 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
4308 .addDef(ResVReg)
4309 .addUse(GR.getSPIRVTypeID(ResType))
4310 .addUse(I.getOperand(2).getReg())
4311 .addUse(I.getOperand(3).getReg())
4312 .addUse(I.getOperand(4).getReg())
4313 .constrainAllUses(TII, TRI, RBI);
4314 return true;
4315}
4316
4317bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
4318 SPIRVTypeInst ResType,
4319 MachineInstr &I) const {
4320 if (getImm(I.getOperand(3), MRI))
4321 return selectExtractVal(ResVReg, ResType, I);
4322 MachineBasicBlock &BB = *I.getParent();
4323 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
4324 .addDef(ResVReg)
4325 .addUse(GR.getSPIRVTypeID(ResType))
4326 .addUse(I.getOperand(2).getReg())
4327 .addUse(I.getOperand(3).getReg())
4328 .constrainAllUses(TII, TRI, RBI);
4329 return true;
4330}
4331
4332bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
4333 SPIRVTypeInst ResType,
4334 MachineInstr &I) const {
4335 const bool IsGEPInBounds = I.getOperand(2).getImm();
4336
4337 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
4338 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
4339 // we have to use Op[InBounds]AccessChain.
4340 const unsigned Opcode = STI.isLogicalSPIRV()
4341 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
4342 : SPIRV::OpAccessChain)
4343 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
4344 : SPIRV::OpPtrAccessChain);
4345
4346 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
4347 .addDef(ResVReg)
4348 .addUse(GR.getSPIRVTypeID(ResType))
4349 // Object to get a pointer to.
4350 .addUse(I.getOperand(3).getReg());
4351 assert(
4352 (Opcode == SPIRV::OpPtrAccessChain ||
4353 Opcode == SPIRV::OpInBoundsPtrAccessChain ||
4354 (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0)) &&
4355 "Cannot translate GEP to OpAccessChain. First index must be 0.");
4356
4357 // Adding indices.
4358 const unsigned StartingIndex =
4359 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
4360 ? 5
4361 : 4;
4362 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
4363 Res.addUse(I.getOperand(i).getReg());
4364 Res.constrainAllUses(TII, TRI, RBI);
4365 return true;
4366}
4367
4368// Maybe wrap a value into OpSpecConstantOp
4369bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
4370 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
4371 unsigned Lim = I.getNumExplicitOperands();
4372 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
4373 Register OpReg = I.getOperand(i).getReg();
4374 MachineInstr *OpDefine = MRI->getVRegDef(OpReg);
4375 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
4376 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
4377 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
4378 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
4379 GR.isAggregateType(OpType)) {
4380 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
4381 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
4382 CompositeArgs.push_back(OpReg);
4383 continue;
4384 }
4385 MachineFunction *MF = I.getMF();
4386 Register WrapReg = GR.find(OpDefine, MF);
4387 if (WrapReg.isValid()) {
4388 CompositeArgs.push_back(WrapReg);
4389 continue;
4390 }
4391 // Create a new register for the wrapper
4392 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType));
4393 CompositeArgs.push_back(WrapReg);
4394 // Decorate the wrapper register and generate a new instruction
4395 MRI->setType(WrapReg, LLT::pointer(0, 64));
4396 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF);
4397 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4398 TII.get(SPIRV::OpSpecConstantOp))
4399 .addDef(WrapReg)
4400 .addUse(GR.getSPIRVTypeID(OpType))
4401 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
4402 .addUse(OpReg);
4403 GR.add(OpDefine, MIB);
4404 MIB.constrainAllUses(TII, TRI, RBI);
4405 }
4406 return true;
4407}
4408
4409bool SPIRVInstructionSelector::selectDerivativeInst(
4410 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
4411 const unsigned DPdOpCode) const {
4412 // TODO: This should check specifically for Fragment Execution Model, but STI
4413 // doesn't provide that information yet. See #167562
4414 errorIfInstrOutsideShader(I);
4415
4416 // If the arg/result types are half then we need to wrap the instr in
4417 // conversions to float
4418 // This case occurs because a half arg/result is legal in HLSL but not spirv.
4419 Register SrcReg = I.getOperand(2).getReg();
4420 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
4421 unsigned BitWidth = std::min(GR.getScalarOrVectorBitWidth(SrcType),
4422 GR.getScalarOrVectorBitWidth(ResType));
4423 if (BitWidth == 32)
4424 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
4425 .addDef(ResVReg)
4426 .addUse(GR.getSPIRVTypeID(ResType))
4427 .addUse(I.getOperand(2).getReg());
4428
4429 MachineIRBuilder MIRBuilder(I);
4430 unsigned componentCount = GR.getScalarOrVectorComponentCount(SrcType);
4431 SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
4432 if (componentCount != 1)
4433 F32ConvertTy = GR.getOrCreateSPIRVVectorType(F32ConvertTy, componentCount,
4434 MIRBuilder, false);
4435
4436 const TargetRegisterClass *RegClass = GR.getRegClass(SrcType);
4437 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
4438 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
4439
4440 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
4441 .addDef(ConvertToVReg)
4442 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
4443 .addUse(SrcReg)
4444 .constrainAllUses(TII, TRI, RBI);
4445 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
4446 .addDef(DpdOpVReg)
4447 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
4448 .addUse(ConvertToVReg)
4449 .constrainAllUses(TII, TRI, RBI);
4450 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
4451 .addDef(ResVReg)
4452 .addUse(GR.getSPIRVTypeID(ResType))
4453 .addUse(DpdOpVReg)
4454 .constrainAllUses(TII, TRI, RBI);
4455 return true;
4456}
4457
4458bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
4459 SPIRVTypeInst ResType,
4460 MachineInstr &I) const {
4461 MachineBasicBlock &BB = *I.getParent();
4462 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
4463 switch (IID) {
4464 case Intrinsic::spv_load:
4465 return selectLoad(ResVReg, ResType, I);
4466 case Intrinsic::spv_store:
4467 return selectStore(I);
4468 case Intrinsic::spv_extractv:
4469 return selectExtractVal(ResVReg, ResType, I);
4470 case Intrinsic::spv_insertv:
4471 return selectInsertVal(ResVReg, ResType, I);
4472 case Intrinsic::spv_extractelt:
4473 return selectExtractElt(ResVReg, ResType, I);
4474 case Intrinsic::spv_insertelt:
4475 return selectInsertElt(ResVReg, ResType, I);
4476 case Intrinsic::spv_gep:
4477 return selectGEP(ResVReg, ResType, I);
4478 case Intrinsic::spv_bitcast: {
4479 Register OpReg = I.getOperand(2).getReg();
4480 SPIRVTypeInst OpType =
4481 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
4482 if (!GR.isBitcastCompatible(ResType, OpType))
4483 report_fatal_error("incompatible result and operand types in a bitcast");
4484 return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
4485 }
4486 case Intrinsic::spv_unref_global:
4487 case Intrinsic::spv_init_global: {
4488 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
4489 MachineInstr *Init = I.getNumExplicitOperands() > 2
4490 ? MRI->getVRegDef(I.getOperand(2).getReg())
4491 : nullptr;
4492 assert(MI);
4493 Register GVarVReg = MI->getOperand(0).getReg();
4494 if (!selectGlobalValue(GVarVReg, *MI, Init))
4495 return false;
4496 // We violate SSA form by inserting OpVariable and still having a gMIR
4497 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
4498 // the duplicated definition.
4499 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
4500 GR.invalidateMachineInstr(MI);
4501 MI->eraseFromParent();
4502 }
4503 return true;
4504 }
4505 case Intrinsic::spv_undef: {
4506 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
4507 .addDef(ResVReg)
4508 .addUse(GR.getSPIRVTypeID(ResType));
4509 MIB.constrainAllUses(TII, TRI, RBI);
4510 return true;
4511 }
4512 case Intrinsic::spv_named_boolean_spec_constant: {
4513 auto Opcode = I.getOperand(3).getImm() ? SPIRV::OpSpecConstantTrue
4514 : SPIRV::OpSpecConstantFalse;
4515
4516 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4517 .addDef(I.getOperand(0).getReg())
4518 .addUse(GR.getSPIRVTypeID(ResType));
4519 MIB.constrainAllUses(TII, TRI, RBI);
4520 unsigned SpecId = I.getOperand(2).getImm();
4521 buildOpDecorate(I.getOperand(0).getReg(), *++MIB->getIterator(), TII,
4522 SPIRV::Decoration::SpecId, {SpecId});
4523
4524 return true;
4525 }
4526 case Intrinsic::spv_const_composite: {
4527 // If no values are attached, the composite is null constant.
4528 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
4529 SmallVector<Register> CompositeArgs;
4530 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
4531
4532 // skip type MD node we already used when generated assign.type for this
4533 if (!IsNull) {
4534 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
4535 return false;
4536 std::function<bool(Register)> HasSpecConstOperand =
4537 [&](Register Reg) -> bool {
4538 MachineInstr *Def = MRI->getVRegDef(Reg);
4539 if (!Def)
4540 return false;
4541 if (!isConstReg(MRI, Def))
4542 return true;
4543 // Recurse into not-yet-selected spv_const_composite intrinsics
4544 // to detect transitive spec constant operands.
4545 if (isSpvIntrinsic(*Def, Intrinsic::spv_const_composite)) {
4546 for (unsigned J = Def->getNumExplicitDefs() + 1;
4547 J < Def->getNumExplicitOperands(); ++J) {
4548 if (Def->getOperand(J).isReg() &&
4549 HasSpecConstOperand(Def->getOperand(J).getReg()))
4550 return true;
4551 }
4552 }
4553 return false;
4554 };
4555 bool HasSpecConst = llvm::any_of(CompositeArgs, HasSpecConstOperand);
4556 unsigned CompositeOpc = HasSpecConst ? SPIRV::OpSpecConstantComposite
4557 : SPIRV::OpConstantComposite;
4558 unsigned ContinuedOpc = HasSpecConst
4559 ? SPIRV::OpSpecConstantCompositeContinuedINTEL
4560 : SPIRV::OpConstantCompositeContinuedINTEL;
4561 MachineIRBuilder MIR(I);
4562 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
4563 MIR, CompositeOpc, 3, ContinuedOpc, CompositeArgs, ResVReg,
4564 GR.getSPIRVTypeID(ResType));
4565 for (auto *Instr : Instructions) {
4566 Instr->setDebugLoc(I.getDebugLoc());
4567 constrainSelectedInstRegOperands(*Instr, TII, TRI, RBI);
4568 }
4569 return true;
4570 } else {
4571 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
4572 .addDef(ResVReg)
4573 .addUse(GR.getSPIRVTypeID(ResType));
4574 MIB.constrainAllUses(TII, TRI, RBI);
4575 return true;
4576 }
4577 }
4578 case Intrinsic::spv_assign_name: {
4579 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
4580 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
4581 for (unsigned i = I.getNumExplicitDefs() + 2;
4582 i < I.getNumExplicitOperands(); ++i) {
4583 MIB.addImm(I.getOperand(i).getImm());
4584 }
4585 MIB.constrainAllUses(TII, TRI, RBI);
4586 return true;
4587 }
4588 case Intrinsic::spv_switch: {
4589 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
4590 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
4591 if (I.getOperand(i).isReg())
4592 MIB.addReg(I.getOperand(i).getReg());
4593 else if (I.getOperand(i).isCImm())
4594 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
4595 else if (I.getOperand(i).isMBB())
4596 MIB.addMBB(I.getOperand(i).getMBB());
4597 else
4598 llvm_unreachable("Unexpected OpSwitch operand");
4599 }
4600 MIB.constrainAllUses(TII, TRI, RBI);
4601 return true;
4602 }
4603 case Intrinsic::spv_loop_merge: {
4604 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge));
4605 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
4606 if (I.getOperand(i).isMBB())
4607 MIB.addMBB(I.getOperand(i).getMBB());
4608 else
4609 MIB.addImm(foldImm(I.getOperand(i), MRI));
4610 }
4611 MIB.constrainAllUses(TII, TRI, RBI);
4612 return true;
4613 }
4614 case Intrinsic::spv_loop_control_intel: {
4615 auto MIB =
4616 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopControlINTEL));
4617 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
4618 MIB.addImm(foldImm(I.getOperand(J), MRI));
4619 MIB.constrainAllUses(TII, TRI, RBI);
4620 return true;
4621 }
4622 case Intrinsic::spv_selection_merge: {
4623 auto MIB =
4624 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge));
4625 assert(I.getOperand(1).isMBB() &&
4626 "operand 1 to spv_selection_merge must be a basic block");
4627 MIB.addMBB(I.getOperand(1).getMBB());
4628 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm()));
4629 MIB.constrainAllUses(TII, TRI, RBI);
4630 return true;
4631 }
4632 case Intrinsic::spv_cmpxchg:
4633 return selectAtomicCmpXchg(ResVReg, ResType, I);
4634 case Intrinsic::spv_unreachable:
4635 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable))
4636 .constrainAllUses(TII, TRI, RBI);
4637 return true;
4638 case Intrinsic::spv_alloca:
4639 return selectFrameIndex(ResVReg, ResType, I);
4640 case Intrinsic::spv_alloca_array:
4641 return selectAllocaArray(ResVReg, ResType, I);
4642 case Intrinsic::spv_assume:
4643 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
4644 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
4645 .addUse(I.getOperand(1).getReg())
4646 .constrainAllUses(TII, TRI, RBI);
4647 return true;
4648 }
4649 break;
4650 case Intrinsic::spv_expect:
4651 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
4652 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
4653 .addDef(ResVReg)
4654 .addUse(GR.getSPIRVTypeID(ResType))
4655 .addUse(I.getOperand(2).getReg())
4656 .addUse(I.getOperand(3).getReg())
4657 .constrainAllUses(TII, TRI, RBI);
4658 return true;
4659 }
4660 break;
4661 case Intrinsic::arithmetic_fence:
4662 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
4663 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT))
4664 .addDef(ResVReg)
4665 .addUse(GR.getSPIRVTypeID(ResType))
4666 .addUse(I.getOperand(2).getReg())
4667 .constrainAllUses(TII, TRI, RBI);
4668 return true;
4669 } else
4670 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I);
4671 break;
4672 case Intrinsic::spv_thread_id:
4673 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
4674 // intrinsic in LLVM IR for SPIR-V backend.
4675 //
4676 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
4677 // `GlobalInvocationId` builtin variable
4678 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
4679 ResType, I);
4680 case Intrinsic::spv_thread_id_in_group:
4681 // The HLSL SV_GroupThreadId semantic is lowered to
4682 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
4683 //
4684 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
4685 // translated to a `LocalInvocationId` builtin variable
4686 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
4687 ResType, I);
4688 case Intrinsic::spv_group_id:
4689 // The HLSL SV_GroupId semantic is lowered to
4690 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
4691 //
4692 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
4693 // builtin variable
4694 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
4695 I);
4696 case Intrinsic::spv_flattened_thread_id_in_group:
4697 // The HLSL SV_GroupIndex semantic is lowered to
4698 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
4699 // backend.
4700 //
4701 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
4702 // a `LocalInvocationIndex` builtin variable
4703 return loadBuiltinInputID(SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
4704 ResType, I);
4705 case Intrinsic::spv_workgroup_size:
4706 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupSize, ResVReg,
4707 ResType, I);
4708 case Intrinsic::spv_global_size:
4709 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
4710 I);
4711 case Intrinsic::spv_global_offset:
4712 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalOffset, ResVReg,
4713 ResType, I);
4714 case Intrinsic::spv_num_workgroups:
4715 return loadVec3BuiltinInputID(SPIRV::BuiltIn::NumWorkgroups, ResVReg,
4716 ResType, I);
4717 case Intrinsic::spv_subgroup_size:
4718 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
4719 I);
4720 case Intrinsic::spv_num_subgroups:
4721 return loadBuiltinInputID(SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
4722 I);
4723 case Intrinsic::spv_subgroup_id:
4724 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
4725 case Intrinsic::spv_subgroup_local_invocation_id:
4726 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupLocalInvocationId,
4727 ResVReg, ResType, I);
4728 case Intrinsic::spv_subgroup_max_size:
4729 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
4730 I);
4731 case Intrinsic::spv_fdot:
4732 return selectFloatDot(ResVReg, ResType, I);
4733 case Intrinsic::spv_udot:
4734 case Intrinsic::spv_sdot:
4735 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4736 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
4737 return selectIntegerDot(ResVReg, ResType, I,
4738 /*Signed=*/IID == Intrinsic::spv_sdot);
4739 return selectIntegerDotExpansion(ResVReg, ResType, I);
4740 case Intrinsic::spv_dot4add_i8packed:
4741 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4742 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
4743 return selectDot4AddPacked<true>(ResVReg, ResType, I);
4744 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
4745 case Intrinsic::spv_dot4add_u8packed:
4746 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4747 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
4748 return selectDot4AddPacked<false>(ResVReg, ResType, I);
4749 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
4750 case Intrinsic::spv_all:
4751 return selectAll(ResVReg, ResType, I);
4752 case Intrinsic::spv_any:
4753 return selectAny(ResVReg, ResType, I);
4754 case Intrinsic::spv_cross:
4755 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross);
4756 case Intrinsic::spv_distance:
4757 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance);
4758 case Intrinsic::spv_lerp:
4759 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix);
4760 case Intrinsic::spv_length:
4761 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length);
4762 case Intrinsic::spv_degrees:
4763 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees);
4764 case Intrinsic::spv_faceforward:
4765 return selectExtInst(ResVReg, ResType, I, GL::FaceForward);
4766 case Intrinsic::spv_frac:
4767 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract);
4768 case Intrinsic::spv_isinf:
4769 return selectOpIsInf(ResVReg, ResType, I);
4770 case Intrinsic::spv_isnan:
4771 return selectOpIsNan(ResVReg, ResType, I);
4772 case Intrinsic::spv_normalize:
4773 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize);
4774 case Intrinsic::spv_refract:
4775 return selectExtInst(ResVReg, ResType, I, GL::Refract);
4776 case Intrinsic::spv_reflect:
4777 return selectExtInst(ResVReg, ResType, I, GL::Reflect);
4778 case Intrinsic::spv_rsqrt:
4779 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
4780 case Intrinsic::spv_sign:
4781 return selectSign(ResVReg, ResType, I);
4782 case Intrinsic::spv_smoothstep:
4783 return selectExtInst(ResVReg, ResType, I, CL::smoothstep, GL::SmoothStep);
4784 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
4785 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
4786 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
4787 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
4788 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
4789 return selectFirstBitLow(ResVReg, ResType, I);
4790 case Intrinsic::spv_group_memory_barrier:
4791 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
4792 SPIRV::MemorySemantics::WorkgroupMemory,
4793 /*WithGroupSync*/ false);
4794 case Intrinsic::spv_group_memory_barrier_with_group_sync:
4795 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
4796 SPIRV::MemorySemantics::WorkgroupMemory,
4797 /*WithGroupSync*/ true);
4798 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
4799 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 1).getReg();
4800 SPIRV::StorageClass::StorageClass ResSC =
4801 GR.getPointerStorageClass(ResType);
4802 if (!isGenericCastablePtr(ResSC))
4803 report_fatal_error("The target storage class is not castable from the "
4804 "Generic storage class");
4805 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGenericCastToPtrExplicit))
4806 .addDef(ResVReg)
4807 .addUse(GR.getSPIRVTypeID(ResType))
4808 .addUse(PtrReg)
4809 .addImm(ResSC)
4810 .constrainAllUses(TII, TRI, RBI);
4811 return true;
4812 }
4813 case Intrinsic::spv_lifetime_start:
4814 case Intrinsic::spv_lifetime_end: {
4815 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
4816 : SPIRV::OpLifetimeStop;
4817 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
4818 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
4819 if (Size == -1)
4820 Size = 0;
4821 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op))
4822 .addUse(PtrReg)
4823 .addImm(Size)
4824 .constrainAllUses(TII, TRI, RBI);
4825 return true;
4826 }
4827 case Intrinsic::spv_saturate:
4828 return selectSaturate(ResVReg, ResType, I);
4829 case Intrinsic::spv_nclamp:
4830 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp);
4831 case Intrinsic::spv_uclamp:
4832 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp);
4833 case Intrinsic::spv_sclamp:
4834 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
4835 case Intrinsic::spv_subgroup_prefix_bit_count:
4836 return selectWavePrefixBitCount(ResVReg, ResType, I);
4837 case Intrinsic::spv_wave_active_countbits:
4838 return selectWaveActiveCountBits(ResVReg, ResType, I);
4839 case Intrinsic::spv_wave_all_equal:
4840 return selectWaveActiveAllEqual(ResVReg, ResType, I);
4841 case Intrinsic::spv_wave_all:
4842 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
4843 case Intrinsic::spv_wave_any:
4844 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
4845 case Intrinsic::spv_subgroup_ballot:
4846 return selectWaveOpInst(ResVReg, ResType, I,
4847 SPIRV::OpGroupNonUniformBallot);
4848 case Intrinsic::spv_wave_is_first_lane:
4849 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect);
4850 case Intrinsic::spv_wave_reduce_or:
4851 return selectWaveReduceOp(ResVReg, ResType, I,
4852 SPIRV::OpGroupNonUniformBitwiseOr);
4853 case Intrinsic::spv_wave_reduce_xor:
4854 return selectWaveReduceOp(ResVReg, ResType, I,
4855 SPIRV::OpGroupNonUniformBitwiseXor);
4856 case Intrinsic::spv_wave_reduce_and:
4857 return selectWaveReduceOp(ResVReg, ResType, I,
4858 SPIRV::OpGroupNonUniformBitwiseAnd);
4859 case Intrinsic::spv_wave_reduce_umax:
4860 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
4861 case Intrinsic::spv_wave_reduce_max:
4862 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
4863 case Intrinsic::spv_wave_reduce_umin:
4864 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
4865 case Intrinsic::spv_wave_reduce_min:
4866 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
4867 case Intrinsic::spv_wave_reduce_sum:
4868 return selectWaveReduceSum(ResVReg, ResType, I);
4869 case Intrinsic::spv_wave_product:
4870 return selectWaveReduceProduct(ResVReg, ResType, I);
4871 case Intrinsic::spv_wave_readlane:
4872 return selectWaveOpInst(ResVReg, ResType, I,
4873 SPIRV::OpGroupNonUniformShuffle);
4874 case Intrinsic::spv_wave_prefix_sum:
4875 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
4876 case Intrinsic::spv_wave_prefix_product:
4877 return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
4878 case Intrinsic::spv_quad_read_across_x: {
4879 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 0);
4880 }
4881 case Intrinsic::spv_quad_read_across_y: {
4882 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 1);
4883 }
4884 case Intrinsic::spv_step:
4885 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step);
4886 case Intrinsic::spv_radians:
4887 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians);
4888 // Discard intrinsics which we do not expect to actually represent code after
4889 // lowering or intrinsics which are not implemented but should not crash when
4890 // found in a customer's LLVM IR input.
4891 case Intrinsic::instrprof_increment:
4892 case Intrinsic::instrprof_increment_step:
4893 case Intrinsic::instrprof_value_profile:
4894 break;
4895 // Discard internal intrinsics.
4896 case Intrinsic::spv_value_md:
4897 break;
4898 case Intrinsic::spv_resource_handlefrombinding: {
4899 return selectHandleFromBinding(ResVReg, ResType, I);
4900 }
4901 case Intrinsic::spv_resource_counterhandlefrombinding:
4902 return selectCounterHandleFromBinding(ResVReg, ResType, I);
4903 case Intrinsic::spv_resource_updatecounter:
4904 return selectUpdateCounter(ResVReg, ResType, I);
4905 case Intrinsic::spv_resource_store_typedbuffer: {
4906 return selectImageWriteIntrinsic(I);
4907 }
4908 case Intrinsic::spv_resource_load_typedbuffer: {
4909 return selectReadImageIntrinsic(ResVReg, ResType, I);
4910 }
4911 case Intrinsic::spv_resource_load_level: {
4912 return selectLoadLevelIntrinsic(ResVReg, ResType, I);
4913 }
4914 case Intrinsic::spv_resource_getdimensions_x:
4915 case Intrinsic::spv_resource_getdimensions_xy:
4916 case Intrinsic::spv_resource_getdimensions_xyz: {
4917 return selectGetDimensionsIntrinsic(ResVReg, ResType, I);
4918 }
4919 case Intrinsic::spv_resource_getdimensions_levels_x:
4920 case Intrinsic::spv_resource_getdimensions_levels_xy:
4921 case Intrinsic::spv_resource_getdimensions_levels_xyz: {
4922 return selectGetDimensionsLevelsIntrinsic(ResVReg, ResType, I);
4923 }
4924 case Intrinsic::spv_resource_getdimensions_ms_xy:
4925 case Intrinsic::spv_resource_getdimensions_ms_xyz: {
4926 return selectGetDimensionsMSIntrinsic(ResVReg, ResType, I);
4927 }
4928 case Intrinsic::spv_resource_calculate_lod:
4929 case Intrinsic::spv_resource_calculate_lod_unclamped:
4930 return selectCalculateLodIntrinsic(ResVReg, ResType, I);
4931 case Intrinsic::spv_resource_sample:
4932 case Intrinsic::spv_resource_sample_clamp:
4933 return selectSampleBasicIntrinsic(ResVReg, ResType, I);
4934 case Intrinsic::spv_resource_samplebias:
4935 case Intrinsic::spv_resource_samplebias_clamp:
4936 return selectSampleBiasIntrinsic(ResVReg, ResType, I);
4937 case Intrinsic::spv_resource_samplegrad:
4938 case Intrinsic::spv_resource_samplegrad_clamp:
4939 return selectSampleGradIntrinsic(ResVReg, ResType, I);
4940 case Intrinsic::spv_resource_samplelevel:
4941 return selectSampleLevelIntrinsic(ResVReg, ResType, I);
4942 case Intrinsic::spv_resource_samplecmp:
4943 case Intrinsic::spv_resource_samplecmp_clamp:
4944 return selectSampleCmpIntrinsic(ResVReg, ResType, I);
4945 case Intrinsic::spv_resource_samplecmplevelzero:
4946 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I);
4947 case Intrinsic::spv_resource_gather:
4948 case Intrinsic::spv_resource_gather_cmp:
4949 return selectGatherIntrinsic(ResVReg, ResType, I);
4950 case Intrinsic::spv_resource_getpointer: {
4951 return selectResourceGetPointer(ResVReg, ResType, I);
4952 }
4953 case Intrinsic::spv_pushconstant_getpointer: {
4954 return selectPushConstantGetPointer(ResVReg, ResType, I);
4955 }
4956 case Intrinsic::spv_discard: {
4957 return selectDiscard(ResVReg, ResType, I);
4958 }
4959 case Intrinsic::spv_resource_nonuniformindex: {
4960 return selectResourceNonUniformIndex(ResVReg, ResType, I);
4961 }
4962 case Intrinsic::spv_unpackhalf2x16: {
4963 return selectExtInst(ResVReg, ResType, I, GL::UnpackHalf2x16);
4964 }
4965 case Intrinsic::spv_packhalf2x16: {
4966 return selectExtInst(ResVReg, ResType, I, GL::PackHalf2x16);
4967 }
4968 case Intrinsic::spv_ddx:
4969 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdx);
4970 case Intrinsic::spv_ddy:
4971 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdy);
4972 case Intrinsic::spv_ddx_coarse:
4973 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxCoarse);
4974 case Intrinsic::spv_ddy_coarse:
4975 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyCoarse);
4976 case Intrinsic::spv_ddx_fine:
4977 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxFine);
4978 case Intrinsic::spv_ddy_fine:
4979 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine);
4980 case Intrinsic::spv_fwidth:
4981 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth);
4982 case Intrinsic::spv_masked_gather:
4983 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
4984 return selectMaskedGather(ResVReg, ResType, I);
4985 return diagnoseUnsupported(
4986 I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
4987 case Intrinsic::spv_masked_scatter:
4988 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
4989 return selectMaskedScatter(I);
4990 return diagnoseUnsupported(
4991 I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
4992 default: {
4993 std::string DiagMsg;
4994 raw_string_ostream OS(DiagMsg);
4995 I.print(OS);
4996 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
4997 report_fatal_error(DiagMsg.c_str(), false);
4998 }
4999 }
5000 return true;
5001}
5002
5003bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
5004 SPIRVTypeInst ResType,
5005 MachineInstr &I) const {
5006 // The images need to be loaded in the same basic block as their use. We defer
5007 // loading the image to the intrinsic that uses it.
5008 if (ResType->getOpcode() == SPIRV::OpTypeImage)
5009 return true;
5010
5011 return loadHandleBeforePosition(ResVReg, GR.getSPIRVTypeForVReg(ResVReg),
5012 *cast<GIntrinsic>(&I), I);
5013}
5014
5015bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
5016 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5017 auto &Intr = cast<GIntrinsic>(I);
5018 assert(Intr.getIntrinsicID() ==
5019 Intrinsic::spv_resource_counterhandlefrombinding);
5020
5021 // Extract information from the intrinsic call.
5022 Register MainHandleReg = Intr.getOperand(2).getReg();
5023 auto *MainHandleDef = cast<GIntrinsic>(getVRegDef(*MRI, MainHandleReg));
5024 assert(MainHandleDef->getIntrinsicID() ==
5025 Intrinsic::spv_resource_handlefrombinding);
5026
5027 uint32_t Set = getIConstVal(Intr.getOperand(4).getReg(), MRI);
5028 uint32_t Binding = getIConstVal(Intr.getOperand(3).getReg(), MRI);
5029 uint32_t ArraySize = getIConstVal(MainHandleDef->getOperand(4).getReg(), MRI);
5030 Register IndexReg = MainHandleDef->getOperand(5).getReg();
5031 std::string CounterName =
5032 getStringValueFromReg(MainHandleDef->getOperand(6).getReg(), *MRI) +
5033 ".counter";
5034
5035 // Create the counter variable.
5036 MachineIRBuilder MIRBuilder(I);
5037 Register CounterVarReg =
5038 buildPointerToResource(SPIRVTypeInst(GR.getPointeeType(ResType)),
5039 GR.getPointerStorageClass(ResType), Set, Binding,
5040 ArraySize, IndexReg, CounterName, MIRBuilder);
5041
5042 return BuildCOPY(ResVReg, CounterVarReg, I);
5043}
5044
5045bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
5046 SPIRVTypeInst ResType,
5047 MachineInstr &I) const {
5048 auto &Intr = cast<GIntrinsic>(I);
5049 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
5050
5051 Register CounterHandleReg = Intr.getOperand(2).getReg();
5052 Register IncrReg = Intr.getOperand(3).getReg();
5053
5054 // The counter handle is a pointer to the counter variable (which is a struct
5055 // containing an i32). We need to get a pointer to that i32 member to do the
5056 // atomic operation.
5057#ifndef NDEBUG
5058 SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
5059 SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType);
5060 assert(CounterVarPointeeType &&
5061 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
5062 "Counter variable must be a struct");
5063 assert(GR.getPointerStorageClass(CounterVarType) ==
5064 SPIRV::StorageClass::StorageBuffer &&
5065 "Counter variable must be in the storage buffer storage class");
5066 assert(CounterVarPointeeType->getNumOperands() == 2 &&
5067 "Counter variable must have exactly 1 member in the struct");
5068 const SPIRVTypeInst MemberType =
5069 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
5070 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
5071 "Counter variable struct must have a single i32 member");
5072#endif
5073
5074 // The struct has a single i32 member.
5075 MachineIRBuilder MIRBuilder(I);
5076 const Type *LLVMIntType =
5077 Type::getInt32Ty(I.getMF()->getFunction().getContext());
5078
5079 SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType(
5080 LLVMIntType, MIRBuilder, SPIRV::StorageClass::StorageBuffer);
5081
5082 Register Zero = buildI32Constant(0, I);
5083
5084 Register PtrToCounter =
5085 MRI->createVirtualRegister(GR.getRegClass(IntPtrType));
5086 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
5087 .addDef(PtrToCounter)
5088 .addUse(GR.getSPIRVTypeID(IntPtrType))
5089 .addUse(CounterHandleReg)
5090 .addUse(Zero)
5091 .constrainAllUses(TII, TRI, RBI);
5092
5093 // For UAV/SSBO counters, the scope is Device. The counter variable is not
5094 // used as a flag. So the memory semantics can be None.
5095 Register Scope = buildI32Constant(SPIRV::Scope::Device, I);
5096 Register Semantics = buildI32Constant(SPIRV::MemorySemantics::None, I);
5097
5098 int64_t IncrVal = getIConstValSext(IncrReg, MRI);
5099 Register Incr = buildI32Constant(static_cast<uint32_t>(IncrVal), I);
5100
5101 Register AtomicRes = MRI->createVirtualRegister(GR.getRegClass(ResType));
5102 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAtomicIAdd))
5103 .addDef(AtomicRes)
5104 .addUse(GR.getSPIRVTypeID(ResType))
5105 .addUse(PtrToCounter)
5106 .addUse(Scope)
5107 .addUse(Semantics)
5108 .addUse(Incr)
5109 .constrainAllUses(TII, TRI, RBI);
5110 if (IncrVal >= 0) {
5111 return BuildCOPY(ResVReg, AtomicRes, I);
5112 }
5113
5114 // In HLSL, IncrementCounter returns the value *before* the increment, while
5115 // DecrementCounter returns the value *after* the decrement. Both are lowered
5116 // to the same atomic intrinsic which returns the value *before* the
5117 // operation. So for decrements (negative IncrVal), we must subtract the
5118 // increment value from the result to get the post-decrement value.
5119 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
5120 .addDef(ResVReg)
5121 .addUse(GR.getSPIRVTypeID(ResType))
5122 .addUse(AtomicRes)
5123 .addUse(Incr)
5124 .constrainAllUses(TII, TRI, RBI);
5125 return true;
5126}
5127bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg,
5128 SPIRVTypeInst ResType,
5129 MachineInstr &I) const {
5130
5131 // If the load of the image is in a different basic block, then
5132 // this will generate invalid code. A proper solution is to move
5133 // the OpLoad from selectHandleFromBinding here. However, to do
5134 // that we will need to change the return type of the intrinsic.
5135 // We will do that when we can, but for now trying to move forward with other
5136 // issues.
5137 Register ImageReg = I.getOperand(2).getReg();
5138 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5139 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5140 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5141 *ImageDef, I)) {
5142 return false;
5143 }
5144
5145 Register IdxReg = I.getOperand(3).getReg();
5146 DebugLoc Loc = I.getDebugLoc();
5147 MachineInstr &Pos = I;
5148
5149 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, IdxReg, Loc,
5150 Pos);
5151}
5152
5153bool SPIRVInstructionSelector::generateSampleImage(
5154 Register ResVReg, SPIRVTypeInst ResType, Register ImageReg,
5155 Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps,
5156 DebugLoc Loc, MachineInstr &Pos) const {
5157 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5158 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5159 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5160 *ImageDef, Pos)) {
5161 return false;
5162 }
5163
5164 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
5165 Register NewSamplerReg =
5166 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
5167 if (!loadHandleBeforePosition(NewSamplerReg,
5168 GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef,
5169 Pos)) {
5170 return false;
5171 }
5172
5173 MachineIRBuilder MIRBuilder(Pos);
5174 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
5175 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
5176 Register SampledImageReg =
5177 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
5178
5179 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpSampledImage))
5180 .addDef(SampledImageReg)
5181 .addUse(GR.getSPIRVTypeID(SampledImageType))
5182 .addUse(NewImageReg)
5183 .addUse(NewSamplerReg)
5184 .constrainAllUses(TII, TRI, RBI);
5185
5186 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
5187 ImOps.Lod.has_value();
5188 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
5189 : SPIRV::OpImageSampleImplicitLod;
5190 if (ImOps.Compare)
5191 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
5192 : SPIRV::OpImageSampleDrefImplicitLod;
5193
5194 auto MIB = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(Opcode))
5195 .addDef(ResVReg)
5196 .addUse(GR.getSPIRVTypeID(ResType))
5197 .addUse(SampledImageReg)
5198 .addUse(CoordinateReg);
5199
5200 if (ImOps.Compare)
5201 MIB.addUse(*ImOps.Compare);
5202
5203 uint32_t ImageOperands = 0;
5204 if (ImOps.Bias)
5205 ImageOperands |= SPIRV::ImageOperand::Bias;
5206 if (ImOps.Lod)
5207 ImageOperands |= SPIRV::ImageOperand::Lod;
5208 if (ImOps.GradX && ImOps.GradY)
5209 ImageOperands |= SPIRV::ImageOperand::Grad;
5210 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(*ImOps.Offset)) {
5211 if (isConstReg(MRI, *ImOps.Offset))
5212 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
5213 else {
5214 Pos.emitGenericError(
5215 "Non-constant offsets are not supported in sample instructions.");
5216 }
5217 }
5218 if (ImOps.MinLod)
5219 ImageOperands |= SPIRV::ImageOperand::MinLod;
5220
5221 if (ImageOperands != 0) {
5222 MIB.addImm(ImageOperands);
5223 if (ImageOperands & SPIRV::ImageOperand::Bias)
5224 MIB.addUse(*ImOps.Bias);
5225 if (ImageOperands & SPIRV::ImageOperand::Lod)
5226 MIB.addUse(*ImOps.Lod);
5227 if (ImageOperands & SPIRV::ImageOperand::Grad) {
5228 MIB.addUse(*ImOps.GradX);
5229 MIB.addUse(*ImOps.GradY);
5230 }
5231 if (ImageOperands &
5232 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
5233 MIB.addUse(*ImOps.Offset);
5234 if (ImageOperands & SPIRV::ImageOperand::MinLod)
5235 MIB.addUse(*ImOps.MinLod);
5236 }
5237
5238 MIB.constrainAllUses(TII, TRI, RBI);
5239 return true;
5240}
5241
5242bool SPIRVInstructionSelector::selectImageQuerySize(
5243 Register ImageReg, Register &ResVReg, MachineInstr &I,
5244 std::optional<Register> LodReg) const {
5245 unsigned Opcode =
5246 LodReg ? SPIRV::OpImageQuerySizeLod : SPIRV::OpImageQuerySize;
5247 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
5248 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
5249 "ImageReg is not an image type.");
5250
5251 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
5252 bool IsArray = ImageType->getOperand(4).getImm() != 0;
5253 unsigned NumComponents = 0;
5254 switch (Dim) {
5255 case SPIRV::Dim::DIM_1D:
5256 case SPIRV::Dim::DIM_Buffer:
5257 NumComponents = IsArray ? 2 : 1;
5258 break;
5259 case SPIRV::Dim::DIM_2D:
5260 case SPIRV::Dim::DIM_Cube:
5261 case SPIRV::Dim::DIM_Rect:
5262 NumComponents = IsArray ? 3 : 2;
5263 break;
5264 case SPIRV::Dim::DIM_3D:
5265 NumComponents = 3;
5266 break;
5267 default:
5268 I.emitGenericError("Unsupported image dimension for OpImageQuerySize.");
5269 return false;
5270 }
5271
5272 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
5273 SPIRVTypeInst ResType =
5274 NumComponents == 1
5275 ? I32Ty
5276 : GR.getOrCreateSPIRVVectorType(I32Ty, NumComponents, I, TII);
5277
5278 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
5279 .addDef(ResVReg)
5280 .addUse(GR.getSPIRVTypeID(ResType))
5281 .addUse(ImageReg);
5282 if (LodReg)
5283 MIB.addUse(*LodReg);
5284 MIB.constrainAllUses(TII, TRI, RBI);
5285 return true;
5286}
5287
5288bool SPIRVInstructionSelector::selectGetDimensionsIntrinsic(
5289 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5290 Register ImageReg = I.getOperand(2).getReg();
5291 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5292 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5293 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5294 *ImageDef, I)) {
5295 return false;
5296 }
5297 return selectImageQuerySize(NewImageReg, ResVReg, I);
5298}
5299
5300bool SPIRVInstructionSelector::selectGetDimensionsLevelsIntrinsic(
5301 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5302 Register ImageReg = I.getOperand(2).getReg();
5303 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5304 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5305 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5306 *ImageDef, I)) {
5307 return false;
5308 }
5309
5310 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
5311 Register LodReg = I.getOperand(3).getReg();
5312
5313 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(6).getImm() == 1 &&
5314 "OpImageQuerySizeLod and OpImageQueryLevels require a sampled image");
5315
5316 if (!selectImageQuerySize(NewImageReg, SizeReg, I, LodReg)) {
5317 return false;
5318 }
5319
5320 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
5321 Register LevelsReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
5322 BuildMI(*I.getParent(), I, I.getDebugLoc(),
5323 TII.get(SPIRV::OpImageQueryLevels))
5324 .addDef(LevelsReg)
5325 .addUse(GR.getSPIRVTypeID(I32Ty))
5326 .addUse(NewImageReg)
5327 .constrainAllUses(TII, TRI, RBI);
5328
5329 BuildMI(*I.getParent(), I, I.getDebugLoc(),
5330 TII.get(SPIRV::OpCompositeConstruct))
5331 .addDef(ResVReg)
5332 .addUse(GR.getSPIRVTypeID(ResType))
5333 .addUse(SizeReg)
5334 .addUse(LevelsReg)
5335 .constrainAllUses(TII, TRI, RBI);
5336
5337 return true;
5338}
5339
5340bool SPIRVInstructionSelector::selectGetDimensionsMSIntrinsic(
5341 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5342 Register ImageReg = I.getOperand(2).getReg();
5343 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5344 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5345 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5346 *ImageDef, I)) {
5347 return false;
5348 }
5349
5350 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
5351
5352 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(5).getImm() == 1 &&
5353 "OpImageQuerySamples requires a multisampled image");
5354
5355 if (!selectImageQuerySize(NewImageReg, SizeReg, I)) {
5356 return false;
5357 }
5358
5359 Register SamplesReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
5360
5361 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
5362 BuildMI(*I.getParent(), I, I.getDebugLoc(),
5363 TII.get(SPIRV::OpImageQuerySamples))
5364 .addDef(SamplesReg)
5365 .addUse(GR.getSPIRVTypeID(I32Ty))
5366 .addUse(NewImageReg)
5367 .constrainAllUses(TII, TRI, RBI);
5368
5369 BuildMI(*I.getParent(), I, I.getDebugLoc(),
5370 TII.get(SPIRV::OpCompositeConstruct))
5371 .addDef(ResVReg)
5372 .addUse(GR.getSPIRVTypeID(ResType))
5373 .addUse(SizeReg)
5374 .addUse(SamplesReg)
5375 .constrainAllUses(TII, TRI, RBI);
5376
5377 return true;
5378}
5379
5380bool SPIRVInstructionSelector::selectCalculateLodIntrinsic(
5381 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5382 Register ImageReg = I.getOperand(2).getReg();
5383 Register SamplerReg = I.getOperand(3).getReg();
5384 Register CoordinateReg = I.getOperand(4).getReg();
5385
5386 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5387 if (!ImageDef)
5388 return false;
5389 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5390 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5391 *ImageDef, I)) {
5392 return false;
5393 }
5394
5395 auto *SamplerDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
5396 if (!SamplerDef)
5397 return false;
5398 Register NewSamplerReg =
5399 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
5400 if (!loadHandleBeforePosition(
5401 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
5402 return false;
5403 }
5404
5405 MachineIRBuilder MIRBuilder(I);
5406 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
5407 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
5408 Register SampledImageReg =
5409 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
5410
5411 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
5412 .addDef(SampledImageReg)
5413 .addUse(GR.getSPIRVTypeID(SampledImageType))
5414 .addUse(NewImageReg)
5415 .addUse(NewSamplerReg)
5416 .constrainAllUses(TII, TRI, RBI);
5417
5418 SPIRVTypeInst Vec2Ty = GR.getOrCreateSPIRVVectorType(ResType, 2, I, TII);
5419 Register QueryResultReg = MRI->createVirtualRegister(GR.getRegClass(Vec2Ty));
5420
5421 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageQueryLod))
5422 .addDef(QueryResultReg)
5423 .addUse(GR.getSPIRVTypeID(Vec2Ty))
5424 .addUse(SampledImageReg)
5425 .addUse(CoordinateReg)
5426 .constrainAllUses(TII, TRI, RBI);
5427
5428 unsigned ExtractedIndex =
5429 cast<GIntrinsic>(I).getIntrinsicID() ==
5430 Intrinsic::spv_resource_calculate_lod_unclamped
5431 ? 1
5432 : 0;
5433
5434 MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
5435 TII.get(SPIRV::OpCompositeExtract))
5436 .addDef(ResVReg)
5437 .addUse(GR.getSPIRVTypeID(ResType))
5438 .addUse(QueryResultReg)
5439 .addImm(ExtractedIndex);
5440
5441 MIB.constrainAllUses(TII, TRI, RBI);
5442 return true;
5443}
5444
5445bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
5446 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5447 Register ImageReg = I.getOperand(2).getReg();
5448 Register SamplerReg = I.getOperand(3).getReg();
5449 Register CoordinateReg = I.getOperand(4).getReg();
5450 ImageOperands ImOps;
5451 if (I.getNumOperands() > 5)
5452 ImOps.Offset = I.getOperand(5).getReg();
5453 if (I.getNumOperands() > 6)
5454 ImOps.MinLod = I.getOperand(6).getReg();
5455 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5456 CoordinateReg, ImOps, I.getDebugLoc(), I);
5457}
5458
5459bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
5460 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5461 Register ImageReg = I.getOperand(2).getReg();
5462 Register SamplerReg = I.getOperand(3).getReg();
5463 Register CoordinateReg = I.getOperand(4).getReg();
5464 ImageOperands ImOps;
5465 ImOps.Bias = I.getOperand(5).getReg();
5466 if (I.getNumOperands() > 6)
5467 ImOps.Offset = I.getOperand(6).getReg();
5468 if (I.getNumOperands() > 7)
5469 ImOps.MinLod = I.getOperand(7).getReg();
5470 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5471 CoordinateReg, ImOps, I.getDebugLoc(), I);
5472}
5473
5474bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
5475 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5476 Register ImageReg = I.getOperand(2).getReg();
5477 Register SamplerReg = I.getOperand(3).getReg();
5478 Register CoordinateReg = I.getOperand(4).getReg();
5479 ImageOperands ImOps;
5480 ImOps.GradX = I.getOperand(5).getReg();
5481 ImOps.GradY = I.getOperand(6).getReg();
5482 if (I.getNumOperands() > 7)
5483 ImOps.Offset = I.getOperand(7).getReg();
5484 if (I.getNumOperands() > 8)
5485 ImOps.MinLod = I.getOperand(8).getReg();
5486 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5487 CoordinateReg, ImOps, I.getDebugLoc(), I);
5488}
5489
5490bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
5491 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5492 Register ImageReg = I.getOperand(2).getReg();
5493 Register SamplerReg = I.getOperand(3).getReg();
5494 Register CoordinateReg = I.getOperand(4).getReg();
5495 ImageOperands ImOps;
5496 ImOps.Lod = I.getOperand(5).getReg();
5497 if (I.getNumOperands() > 6)
5498 ImOps.Offset = I.getOperand(6).getReg();
5499 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5500 CoordinateReg, ImOps, I.getDebugLoc(), I);
5501}
5502
5503bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg,
5504 SPIRVTypeInst ResType,
5505 MachineInstr &I) const {
5506 Register ImageReg = I.getOperand(2).getReg();
5507 Register SamplerReg = I.getOperand(3).getReg();
5508 Register CoordinateReg = I.getOperand(4).getReg();
5509 ImageOperands ImOps;
5510 ImOps.Compare = I.getOperand(5).getReg();
5511 if (I.getNumOperands() > 6)
5512 ImOps.Offset = I.getOperand(6).getReg();
5513 if (I.getNumOperands() > 7)
5514 ImOps.MinLod = I.getOperand(7).getReg();
5515 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5516 CoordinateReg, ImOps, I.getDebugLoc(), I);
5517}
5518
5519bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(Register &ResVReg,
5520 SPIRVTypeInst ResType,
5521 MachineInstr &I) const {
5522 Register ImageReg = I.getOperand(2).getReg();
5523 Register CoordinateReg = I.getOperand(3).getReg();
5524 Register LodReg = I.getOperand(4).getReg();
5525
5526 ImageOperands ImOps;
5527 ImOps.Lod = LodReg;
5528 if (I.getNumOperands() > 5)
5529 ImOps.Offset = I.getOperand(5).getReg();
5530
5531 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5532 if (!ImageDef)
5533 return false;
5534
5535 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5536 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5537 *ImageDef, I)) {
5538 return false;
5539 }
5540
5541 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, CoordinateReg,
5542 I.getDebugLoc(), I, &ImOps);
5543}
5544
5545bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
5546 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5547 Register ImageReg = I.getOperand(2).getReg();
5548 Register SamplerReg = I.getOperand(3).getReg();
5549 Register CoordinateReg = I.getOperand(4).getReg();
5550 ImageOperands ImOps;
5551 ImOps.Compare = I.getOperand(5).getReg();
5552 if (I.getNumOperands() > 6)
5553 ImOps.Offset = I.getOperand(6).getReg();
5554 SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
5555 ImOps.Lod = GR.getOrCreateConstFP(APFloat(0.0f), I, FloatTy, TII);
5556 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
5557 CoordinateReg, ImOps, I.getDebugLoc(), I);
5558}
5559
5560bool SPIRVInstructionSelector::selectGatherIntrinsic(Register &ResVReg,
5561 SPIRVTypeInst ResType,
5562 MachineInstr &I) const {
5563 Register ImageReg = I.getOperand(2).getReg();
5564 Register SamplerReg = I.getOperand(3).getReg();
5565 Register CoordinateReg = I.getOperand(4).getReg();
5566 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
5567 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
5568 "ImageReg is not an image type.");
5569
5570 Register ComponentOrCompareReg;
5571 Register OffsetReg;
5572
5573 ComponentOrCompareReg = I.getOperand(5).getReg();
5574 OffsetReg = I.getOperand(6).getReg();
5575 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5576 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5577 if (!loadHandleBeforePosition(NewImageReg, ImageType, *ImageDef, I)) {
5578 return false;
5579 }
5580
5581 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
5582 if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube &&
5583 Dim != SPIRV::Dim::DIM_Rect) {
5584 I.emitGenericError(
5585 "Gather operations are only supported for 2D, Cube, and Rect images.");
5586 return false;
5587 }
5588
5589 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
5590 Register NewSamplerReg =
5591 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
5592 if (!loadHandleBeforePosition(
5593 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
5594 return false;
5595 }
5596
5597 MachineIRBuilder MIRBuilder(I);
5598 SPIRVTypeInst SampledImageType =
5599 GR.getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
5600 Register SampledImageReg =
5601 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
5602
5603 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
5604 .addDef(SampledImageReg)
5605 .addUse(GR.getSPIRVTypeID(SampledImageType))
5606 .addUse(NewImageReg)
5607 .addUse(NewSamplerReg)
5608 .constrainAllUses(TII, TRI, RBI);
5609
5610 auto IntrId = cast<GIntrinsic>(I).getIntrinsicID();
5611 bool IsGatherCmp = IntrId == Intrinsic::spv_resource_gather_cmp;
5612 unsigned Opcode =
5613 IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather;
5614
5615 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
5616 .addDef(ResVReg)
5617 .addUse(GR.getSPIRVTypeID(ResType))
5618 .addUse(SampledImageReg)
5619 .addUse(CoordinateReg)
5620 .addUse(ComponentOrCompareReg);
5621
5622 uint32_t ImageOperands = 0;
5623 if (OffsetReg && !isScalarOrVectorIntConstantZero(OffsetReg)) {
5624 if (Dim == SPIRV::Dim::DIM_Cube) {
5625 I.emitGenericError(
5626 "Gather operations with offset are not supported for Cube images.");
5627 return false;
5628 }
5629 if (isConstReg(MRI, OffsetReg))
5630 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
5631 else {
5632 ImageOperands |= SPIRV::ImageOperand::Offset;
5633 }
5634 }
5635
5636 if (ImageOperands != 0) {
5637 MIB.addImm(ImageOperands);
5638 if (ImageOperands &
5639 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
5640 MIB.addUse(OffsetReg);
5641 }
5642
5643 MIB.constrainAllUses(TII, TRI, RBI);
5644 return true;
5645}
5646
5647bool SPIRVInstructionSelector::generateImageReadOrFetch(
5648 Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg,
5649 Register IdxReg, DebugLoc Loc, MachineInstr &Pos,
5650 const ImageOperands *ImOps) const {
5651 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
5652 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
5653 "ImageReg is not an image type.");
5654
5655 bool IsSignedInteger =
5656 sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType));
5657 // Check if the "sampled" operand of the image type is 1.
5658 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
5659 auto SampledOp = ImageType->getOperand(6);
5660 bool IsFetch = (SampledOp.getImm() == 1);
5661
5662 auto AddOperands = [&](MachineInstrBuilder &MIB) {
5663 uint32_t ImageOperandsMask = 0;
5664 if (IsSignedInteger)
5665 ImageOperandsMask |= 0x1000; // SignExtend
5666
5667 if (IsFetch && ImOps) {
5668 if (ImOps->Lod)
5669 ImageOperandsMask |= SPIRV::ImageOperand::Lod;
5670 if (ImOps->Offset && !isScalarOrVectorIntConstantZero(*ImOps->Offset)) {
5671 if (isConstReg(MRI, *ImOps->Offset))
5672 ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset;
5673 else
5674 ImageOperandsMask |= SPIRV::ImageOperand::Offset;
5675 }
5676 }
5677
5678 if (ImageOperandsMask != 0) {
5679 MIB.addImm(ImageOperandsMask);
5680 if (IsFetch && ImOps) {
5681 if (ImOps->Lod)
5682 MIB.addUse(*ImOps->Lod);
5683 if (ImOps->Offset &&
5684 (ImageOperandsMask &
5685 (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset)))
5686 MIB.addUse(*ImOps->Offset);
5687 }
5688 }
5689 };
5690
5691 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
5692 if (ResultSize == 4) {
5693 auto BMI =
5694 BuildMI(*Pos.getParent(), Pos, Loc,
5695 TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
5696 .addDef(ResVReg)
5697 .addUse(GR.getSPIRVTypeID(ResType))
5698 .addUse(ImageReg)
5699 .addUse(IdxReg);
5700
5701 AddOperands(BMI);
5702 BMI.constrainAllUses(TII, TRI, RBI);
5703 return true;
5704 }
5705
5706 SPIRVTypeInst ReadType = widenTypeToVec4(ResType, Pos);
5707 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType));
5708 auto BMI =
5709 BuildMI(*Pos.getParent(), Pos, Loc,
5710 TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
5711 .addDef(ReadReg)
5712 .addUse(GR.getSPIRVTypeID(ReadType))
5713 .addUse(ImageReg)
5714 .addUse(IdxReg);
5715 AddOperands(BMI);
5716 BMI.constrainAllUses(TII, TRI, RBI);
5717
5718 if (ResultSize == 1) {
5719 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpCompositeExtract))
5720 .addDef(ResVReg)
5721 .addUse(GR.getSPIRVTypeID(ResType))
5722 .addUse(ReadReg)
5723 .addImm(0)
5724 .constrainAllUses(TII, TRI, RBI);
5725 return true;
5726 }
5727 return extractSubvector(ResVReg, ResType, ReadReg, Pos);
5728}
5729
5730bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg,
5731 SPIRVTypeInst ResType,
5732 MachineInstr &I) const {
5733 Register ResourcePtr = I.getOperand(2).getReg();
5734 SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(ResourcePtr, I.getMF());
5735 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
5736 // For texel buffers, the index into the image is part of the OpImageRead or
5737 // OpImageWrite instructions. So we will do nothing in this case. This
5738 // intrinsic will be combined with the load or store when selecting the load
5739 // or store.
5740 return true;
5741 }
5742
5743 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
5744 MachineIRBuilder MIRBuilder(I);
5745
5746 Register IndexReg = I.getOperand(3).getReg();
5747 Register ZeroReg =
5748 buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
5749 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
5750 .addDef(ResVReg)
5751 .addUse(GR.getSPIRVTypeID(ResType))
5752 .addUse(ResourcePtr)
5753 .addUse(ZeroReg)
5754 .addUse(IndexReg)
5755 .constrainAllUses(TII, TRI, RBI);
5756 return true;
5757}
5758
5759bool SPIRVInstructionSelector::selectPushConstantGetPointer(
5760 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5761 MRI->replaceRegWith(ResVReg, I.getOperand(2).getReg());
5762 return true;
5763}
5764
5765bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
5766 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5767 Register ObjReg = I.getOperand(2).getReg();
5768 if (!BuildCOPY(ResVReg, ObjReg, I))
5769 return false;
5770
5771 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {});
5772 // Check for the registers that use the index marked as non-uniform
5773 // and recursively mark them as non-uniform.
5774 // Per the spec, it's necessary that the final argument used for
5775 // load/store/sample/atomic must be decorated, so we need to propagate the
5776 // decoration through access chains and copies.
5777 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
5778 decorateUsesAsNonUniform(ResVReg);
5779 return true;
5780}
5781
5782void SPIRVInstructionSelector::decorateUsesAsNonUniform(
5783 Register &NonUniformReg) const {
5784 llvm::SmallVector<Register> WorkList = {NonUniformReg};
5785 while (WorkList.size() > 0) {
5786 Register CurrentReg = WorkList.back();
5787 WorkList.pop_back();
5788
5789 bool IsDecorated = false;
5790 for (MachineInstr &Use : MRI->use_instructions(CurrentReg)) {
5791 if (Use.getOpcode() == SPIRV::OpDecorate &&
5792 Use.getOperand(1).getImm() == SPIRV::Decoration::NonUniformEXT) {
5793 IsDecorated = true;
5794 continue;
5795 }
5796 // Check if the instruction has the result register and add it to the
5797 // worklist.
5798 if (Use.getOperand(0).isReg() && Use.getOperand(0).isDef()) {
5799 Register ResultReg = Use.getOperand(0).getReg();
5800 if (ResultReg == CurrentReg)
5801 continue;
5802 WorkList.push_back(ResultReg);
5803 }
5804 }
5805
5806 if (!IsDecorated) {
5807 buildOpDecorate(CurrentReg, *MRI->getVRegDef(CurrentReg), TII,
5808 SPIRV::Decoration::NonUniformEXT, {});
5809 }
5810 }
5811}
5812
5813bool SPIRVInstructionSelector::extractSubvector(
5814 Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg,
5815 MachineInstr &InsertionPoint) const {
5816 SPIRVTypeInst InputType = GR.getResultType(ReadReg);
5817 [[maybe_unused]] uint64_t InputSize =
5818 GR.getScalarOrVectorComponentCount(InputType);
5819 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
5820 assert(InputSize > 1 && "The input must be a vector.");
5821 assert(ResultSize > 1 && "The result must be a vector.");
5822 assert(ResultSize < InputSize &&
5823 "Cannot extract more element than there are in the input.");
5824 SmallVector<Register> ComponentRegisters;
5825 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(ResType);
5826 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType);
5827 for (uint64_t I = 0; I < ResultSize; I++) {
5828 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass);
5829 BuildMI(*InsertionPoint.getParent(), InsertionPoint,
5830 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
5831 .addDef(ComponentReg)
5832 .addUse(ScalarType->getOperand(0).getReg())
5833 .addUse(ReadReg)
5834 .addImm(I)
5835 .constrainAllUses(TII, TRI, RBI);
5836 ComponentRegisters.emplace_back(ComponentReg);
5837 }
5838
5839 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint,
5840 InsertionPoint.getDebugLoc(),
5841 TII.get(SPIRV::OpCompositeConstruct))
5842 .addDef(ResVReg)
5843 .addUse(GR.getSPIRVTypeID(ResType));
5844
5845 for (Register ComponentReg : ComponentRegisters)
5846 MIB.addUse(ComponentReg);
5847 MIB.constrainAllUses(TII, TRI, RBI);
5848 return true;
5849}
5850
5851bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
5852 MachineInstr &I) const {
5853 // If the load of the image is in a different basic block, then
5854 // this will generate invalid code. A proper solution is to move
5855 // the OpLoad from selectHandleFromBinding here. However, to do
5856 // that we will need to change the return type of the intrinsic.
5857 // We will do that when we can, but for now trying to move forward with other
5858 // issues.
5859 Register ImageReg = I.getOperand(1).getReg();
5860 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
5861 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
5862 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
5863 *ImageDef, I)) {
5864 return false;
5865 }
5866
5867 Register CoordinateReg = I.getOperand(2).getReg();
5868 Register DataReg = I.getOperand(3).getReg();
5869 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
5870 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
5871 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite))
5872 .addUse(NewImageReg)
5873 .addUse(CoordinateReg)
5874 .addUse(DataReg)
5875 .constrainAllUses(TII, TRI, RBI);
5876 return true;
5877}
5878
5879Register SPIRVInstructionSelector::buildPointerToResource(
5880 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
5881 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
5882 StringRef Name, MachineIRBuilder MIRBuilder) const {
5883 const Type *ResType = GR.getTypeForSPIRVType(SpirvResType);
5884 if (ArraySize == 1) {
5885 SPIRVTypeInst PtrType =
5886 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
5887 assert(GR.getPointeeType(PtrType) == SpirvResType &&
5888 "SpirvResType did not have an explicit layout.");
5889 return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding, Name,
5890 MIRBuilder);
5891 }
5892
5893 const Type *VarType = ArrayType::get(const_cast<Type *>(ResType), ArraySize);
5894 SPIRVTypeInst VarPointerType =
5895 GR.getOrCreateSPIRVPointerType(VarType, MIRBuilder, SC);
5896 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
5897 VarPointerType, Set, Binding, Name, MIRBuilder);
5898
5899 SPIRVTypeInst ResPointerType =
5900 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
5901 Register AcReg = MRI->createVirtualRegister(GR.getRegClass(ResPointerType));
5902
5903 MIRBuilder.buildInstr(SPIRV::OpAccessChain)
5904 .addDef(AcReg)
5905 .addUse(GR.getSPIRVTypeID(ResPointerType))
5906 .addUse(VarReg)
5907 .addUse(IndexReg);
5908
5909 return AcReg;
5910}
5911
5912bool SPIRVInstructionSelector::selectFirstBitSet16(
5913 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
5914 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
5915 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
5916 if (!selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()},
5917 ExtendOpcode))
5918 return false;
5919
5920 return selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode);
5921}
5922
5923bool SPIRVInstructionSelector::selectFirstBitSet32(
5924 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
5925 unsigned BitSetOpcode) const {
5926 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
5927 .addDef(ResVReg)
5928 .addUse(GR.getSPIRVTypeID(ResType))
5929 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
5930 .addImm(BitSetOpcode)
5931 .addUse(SrcReg)
5932 .constrainAllUses(TII, TRI, RBI);
5933 return true;
5934}
5935
5936bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
5937 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
5938 unsigned BitSetOpcode, bool SwapPrimarySide) const {
5939
5940 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
5941 // requires creating a param register and return register with an invalid
5942 // vector size. If that is resolved, then this function can be used for
5943 // vectors of any component size.
5944 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
5945 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
5946
5947 MachineIRBuilder MIRBuilder(I);
5948 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
5949 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder);
5950 SPIRVTypeInst I64x2Type =
5951 GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder, false);
5952 SPIRVTypeInst Vec2ResType =
5953 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder, false);
5954
5955 std::vector<Register> PartialRegs;
5956
5957 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
5958 unsigned CurrentComponent = 0;
5959 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
5960 // This register holds the firstbitX result for each of the i64x2 vectors
5961 // extracted from SrcReg
5962 Register BitSetResult =
5963 MRI->createVirtualRegister(GR.getRegClass(I64x2Type));
5964
5965 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
5966 TII.get(SPIRV::OpVectorShuffle))
5967 .addDef(BitSetResult)
5968 .addUse(GR.getSPIRVTypeID(I64x2Type))
5969 .addUse(SrcReg)
5970 .addUse(SrcReg)
5971 .addImm(CurrentComponent)
5972 .addImm(CurrentComponent + 1);
5973
5974 MIB.constrainAllUses(TII, TRI, RBI);
5975
5976 Register SubVecBitSetReg =
5977 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType));
5978
5979 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult,
5980 BitSetOpcode, SwapPrimarySide))
5981 return false;
5982
5983 PartialRegs.push_back(SubVecBitSetReg);
5984 }
5985
5986 // On odd component counts we need to handle one more component
5987 if (CurrentComponent != ComponentCount) {
5988 bool ZeroAsNull = !STI.isShader();
5989 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
5990 Register ConstIntLastIdx = GR.getOrCreateConstInt(
5991 ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
5992
5993 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx},
5994 SPIRV::OpVectorExtractDynamic))
5995 return false;
5996
5997 Register FinalElemBitSetReg =
5998 MRI->createVirtualRegister(GR.getRegClass(BaseType));
5999
6000 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg,
6001 BitSetOpcode, SwapPrimarySide))
6002 return false;
6003
6004 PartialRegs.push_back(FinalElemBitSetReg);
6005 }
6006
6007 // Join all the resulting registers back into the return type in order
6008 // (ie i32x2, i32x2, i32x1 -> i32x5)
6009 return selectOpWithSrcs(ResVReg, ResType, I, std::move(PartialRegs),
6010 SPIRV::OpCompositeConstruct);
6011}
6012
6013bool SPIRVInstructionSelector::selectFirstBitSet64(
6014 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
6015 unsigned BitSetOpcode, bool SwapPrimarySide) const {
6016 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
6017 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
6018 bool ZeroAsNull = !STI.isShader();
6019 Register ConstIntZero =
6020 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull);
6021 Register ConstIntOne =
6022 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull);
6023
6024 // SPIRV doesn't support vectors with more than 4 components. Since the
6025 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
6026 // operate on vectors with 2 or less components. When largers vectors are
6027 // seen. Split them, recurse, then recombine them.
6028 if (ComponentCount > 2) {
6029 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
6030 BitSetOpcode, SwapPrimarySide);
6031 }
6032
6033 // 1. Split int64 into 2 pieces using a bitcast
6034 MachineIRBuilder MIRBuilder(I);
6035 SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType(
6036 BaseType, 2 * ComponentCount, MIRBuilder, false);
6037 Register BitcastReg =
6038 MRI->createVirtualRegister(GR.getRegClass(PostCastType));
6039
6040 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg},
6041 SPIRV::OpBitcast))
6042 return false;
6043
6044 // 2. Find the first set bit from the primary side for all the pieces in #1
6045 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType));
6046 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode))
6047 return false;
6048
6049 // 3. Split result vector into high bits and low bits
6050 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6051 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6052
6053 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
6054 if (IsScalarRes) {
6055 // if scalar do a vector extract
6056 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntOne},
6057 SPIRV::OpVectorExtractDynamic))
6058 return false;
6059 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntZero},
6060 SPIRV::OpVectorExtractDynamic))
6061 return false;
6062 } else {
6063 // if vector do a shufflevector
6064 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6065 TII.get(SPIRV::OpVectorShuffle))
6066 .addDef(HighReg)
6067 .addUse(GR.getSPIRVTypeID(ResType))
6068 .addUse(FBSReg)
6069 // Per the spec, repeat the vector if only one vec is needed
6070 .addUse(FBSReg);
6071
6072 // high bits are stored in even natural indexes. Extract them from FBSReg
6073 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
6074 MIB.addImm(J);
6075 }
6076
6077 MIB.constrainAllUses(TII, TRI, RBI);
6078
6079 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6080 TII.get(SPIRV::OpVectorShuffle))
6081 .addDef(LowReg)
6082 .addUse(GR.getSPIRVTypeID(ResType))
6083 .addUse(FBSReg)
6084 // Per the spec, repeat the vector if only one vec is needed
6085 .addUse(FBSReg);
6086
6087 // low bits are stored in odd natural indices. Extract them from FBSReg
6088 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
6089 MIB.addImm(J);
6090 }
6091 MIB.constrainAllUses(TII, TRI, RBI);
6092 }
6093
6094 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
6095 // primary
6096 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
6097 Register NegOneReg;
6098 Register Reg0;
6099 Register Reg32;
6100 unsigned SelectOp;
6101 unsigned AddOp;
6102
6103 if (IsScalarRes) {
6104 NegOneReg =
6105 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull);
6106 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
6107 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull);
6108 SelectOp = SPIRV::OpSelectSISCond;
6109 AddOp = SPIRV::OpIAddS;
6110 } else {
6111 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount,
6112 MIRBuilder, false);
6113 NegOneReg =
6114 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull);
6115 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull);
6116 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull);
6117 SelectOp = SPIRV::OpSelectVIVCond;
6118 AddOp = SPIRV::OpIAddV;
6119 }
6120
6121 Register PrimaryReg = HighReg;
6122 Register SecondaryReg = LowReg;
6123 Register RegPrimaryOffset = Reg32;
6124 Register RegSecondaryOffset = Reg0;
6125
6126 // By default the emitted opcodes check for the set bit from the MSB side.
6127 // Setting SwapPrimarySide checks the set bit from the LSB side
6128 if (SwapPrimarySide) {
6129 PrimaryReg = LowReg;
6130 SecondaryReg = HighReg;
6131 RegPrimaryOffset = Reg0;
6132 RegSecondaryOffset = Reg32;
6133 }
6134
6135 Register RegSecondaryHasVal =
6136 MRI->createVirtualRegister(GR.getRegClass(BoolType));
6137 if (!selectOpWithSrcs(RegSecondaryHasVal, BoolType, I,
6138 {SecondaryReg, NegOneReg}, SPIRV::OpINotEqual))
6139 return false;
6140
6141 Register RegPrimaryHasVal =
6142 MRI->createVirtualRegister(GR.getRegClass(BoolType));
6143 if (!selectOpWithSrcs(RegPrimaryHasVal, BoolType, I, {PrimaryReg, NegOneReg},
6144 SPIRV::OpINotEqual))
6145 return false;
6146
6147 // Pass 1: seed with secondary (lower-priority fallback)
6148 // ReturnBits = secondaryHasVal ? SecondaryBits : -1
6149 // Add = secondaryHasVal ? SecondaryOffset : 0
6150 Register RegReturnBits = MRI->createVirtualRegister(GR.getRegClass(ResType));
6151 if (!selectOpWithSrcs(RegReturnBits, ResType, I,
6152 {RegSecondaryHasVal, SecondaryReg, NegOneReg},
6153 SelectOp))
6154 return false;
6155
6156 Register RegAdd;
6157 if (SwapPrimarySide) {
6158 RegAdd = MRI->createVirtualRegister(GR.getRegClass(ResType));
6159 if (!selectOpWithSrcs(RegAdd, ResType, I,
6160 {RegSecondaryHasVal, RegSecondaryOffset, Reg0},
6161 SelectOp))
6162 return false;
6163 } else {
6164 RegAdd = Reg0;
6165 }
6166
6167 // Pass 2: override with primary (higher priority) if it has a valid result
6168 // ReturnBits2 = primaryHasVal ? PrimaryBits : ReturnBits
6169 // Add2 = primaryHasVal ? PrimaryOffset : Add
6170 Register RegReturnBits2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
6171 if (!selectOpWithSrcs(RegReturnBits2, ResType, I,
6172 {RegPrimaryHasVal, PrimaryReg, RegReturnBits},
6173 SelectOp))
6174 return false;
6175
6176 Register RegAdd2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
6177 if (!selectOpWithSrcs(RegAdd2, ResType, I,
6178 {RegPrimaryHasVal, RegPrimaryOffset, RegAdd}, SelectOp))
6179 return false;
6180
6181 return selectOpWithSrcs(ResVReg, ResType, I, {RegReturnBits2, RegAdd2},
6182 AddOp);
6183}
6184
6185bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
6186 SPIRVTypeInst ResType,
6187 MachineInstr &I,
6188 bool IsSigned) const {
6189 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
6190 Register OpReg = I.getOperand(2).getReg();
6191 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
6192 // zero or sign extend
6193 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
6194 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
6195
6196 switch (GR.getScalarOrVectorBitWidth(OpType)) {
6197 case 16:
6198 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
6199 case 32:
6200 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
6201 case 64:
6202 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
6203 /*SwapPrimarySide=*/false);
6204 default:
6205 report_fatal_error(
6206 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
6207 }
6208}
6209
6210bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
6211 SPIRVTypeInst ResType,
6212 MachineInstr &I) const {
6213 // FindILsb intrinsic only supports 32 bit integers
6214 Register OpReg = I.getOperand(2).getReg();
6215 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
6216 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
6217 // to an unsigned i32. As this leaves all the least significant bits unchanged
6218 // so the first set bit from the LSB side doesn't change.
6219 unsigned ExtendOpcode = SPIRV::OpUConvert;
6220 unsigned BitSetOpcode = GL::FindILsb;
6221
6222 switch (GR.getScalarOrVectorBitWidth(OpType)) {
6223 case 16:
6224 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
6225 case 32:
6226 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
6227 case 64:
6228 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
6229 /*SwapPrimarySide=*/true);
6230 default:
6231 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits.");
6232 }
6233}
6234
6235bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
6236 SPIRVTypeInst ResType,
6237 MachineInstr &I) const {
6238 // there was an allocation size parameter to the allocation instruction
6239 // that is not 1
6240 MachineBasicBlock &BB = *I.getParent();
6241 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVariableLengthArrayINTEL))
6242 .addDef(ResVReg)
6243 .addUse(GR.getSPIRVTypeID(ResType))
6244 .addUse(I.getOperand(2).getReg())
6245 .constrainAllUses(TII, TRI, RBI);
6246 if (!STI.isShader()) {
6247 unsigned Alignment = I.getOperand(3).getImm();
6248 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
6249 }
6250 return true;
6251}
6252
6253bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
6254 SPIRVTypeInst ResType,
6255 MachineInstr &I) const {
6256 // Change order of instructions if needed: all OpVariable instructions in a
6257 // function must be the first instructions in the first block
6258 auto It = getOpVariableMBBIt(I);
6259 BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
6260 .addDef(ResVReg)
6261 .addUse(GR.getSPIRVTypeID(ResType))
6262 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
6263 .constrainAllUses(TII, TRI, RBI);
6264 if (!STI.isShader()) {
6265 unsigned Alignment = I.getOperand(2).getImm();
6266 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
6267 {Alignment});
6268 }
6269 return true;
6270}
6271
6272bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
6273 // InstructionSelector walks backwards through the instructions. We can use
6274 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
6275 // first, so can generate an OpBranchConditional here. If there is no
6276 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
6277 const MachineInstr *PrevI = I.getPrevNode();
6278 MachineBasicBlock &MBB = *I.getParent();
6279 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
6280 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
6281 .addUse(PrevI->getOperand(0).getReg())
6282 .addMBB(PrevI->getOperand(1).getMBB())
6283 .addMBB(I.getOperand(0).getMBB())
6284 .constrainAllUses(TII, TRI, RBI);
6285 return true;
6286 }
6287 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
6288 .addMBB(I.getOperand(0).getMBB())
6289 .constrainAllUses(TII, TRI, RBI);
6290 return true;
6291}
6292
6293bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
6294 // InstructionSelector walks backwards through the instructions. For an
6295 // explicit conditional branch with no fallthrough, we use both a G_BR and a
6296 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
6297 // generate the OpBranchConditional in selectBranch above.
6298 //
6299 // If an OpBranchConditional has been generated, we simply return, as the work
6300 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
6301 // implicit fallthrough to the next basic block, so we need to create an
6302 // OpBranchConditional with an explicit "false" argument pointing to the next
6303 // basic block that LLVM would fall through to.
6304 const MachineInstr *NextI = I.getNextNode();
6305 // Check if this has already been successfully selected.
6306 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
6307 return true;
6308 // Must be relying on implicit block fallthrough, so generate an
6309 // OpBranchConditional with the "next" basic block as the "false" target.
6310 MachineBasicBlock &MBB = *I.getParent();
6311 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
6312 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
6313 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
6314 .addUse(I.getOperand(0).getReg())
6315 .addMBB(I.getOperand(1).getMBB())
6316 .addMBB(NextMBB)
6317 .constrainAllUses(TII, TRI, RBI);
6318 return true;
6319}
6320
6321bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
6322 MachineInstr &I) const {
6323 auto MIB =
6324 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::PHI))
6325 .addDef(ResVReg);
6326 const unsigned NumOps = I.getNumOperands();
6327 for (unsigned i = 1; i < NumOps; i += 2) {
6328 MIB.addUse(I.getOperand(i + 0).getReg());
6329 MIB.addMBB(I.getOperand(i + 1).getMBB());
6330 }
6331 MIB.constrainAllUses(TII, TRI, RBI);
6332 return true;
6333}
6334
6335bool SPIRVInstructionSelector::selectGlobalValue(
6336 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
6337 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
6338 MachineIRBuilder MIRBuilder(I);
6339 const GlobalValue *GV = I.getOperand(1).getGlobal();
6340 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV));
6341
6342 std::string GlobalIdent;
6343 if (!GV->hasName()) {
6344 unsigned &ID = UnnamedGlobalIDs[GV];
6345 if (ID == 0)
6346 ID = UnnamedGlobalIDs.size();
6347 GlobalIdent = "__unnamed_" + Twine(ID).str();
6348 } else {
6349 GlobalIdent = GV->getName();
6350 }
6351
6352 // Behaviour of functions as operands depends on availability of the
6353 // corresponding extension (SPV_INTEL_function_pointers):
6354 // - If there is an extension to operate with functions as operands:
6355 // We create a proper constant operand and evaluate a correct type for a
6356 // function pointer.
6357 // - Without the required extension:
6358 // We have functions as operands in tests with blocks of instruction e.g. in
6359 // transcoding/global_block.ll. These operands are not used and should be
6360 // substituted by zero constants. Their type is expected to be always
6361 // OpTypePointer Function %uchar.
6362 if (isa<Function>(GV)) {
6363 const Constant *ConstVal = GV;
6364 MachineBasicBlock &BB = *I.getParent();
6365 Register NewReg = GR.find(ConstVal, GR.CurMF);
6366 if (!NewReg.isValid()) {
6367 const Function *GVFun =
6368 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
6369 ? dyn_cast<Function>(GV)
6370 : nullptr;
6371 SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType(
6372 GVType, I,
6373 GVFun ? SPIRV::StorageClass::CodeSectionINTEL
6374 : addressSpaceToStorageClass(GV->getAddressSpace(), STI));
6375 if (GVFun) {
6376 // References to a function via function pointers generate virtual
6377 // registers without a definition. We will resolve it later, during
6378 // module analysis stage.
6379 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
6380 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
6381 Register FuncVReg =
6382 MRI->createGenericVirtualRegister(GR.getRegType(ResType));
6383 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass);
6384 GR.assignSPIRVTypeToVReg(ResType, FuncVReg, *GR.CurMF);
6385 MachineInstrBuilder MIB1 =
6386 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
6387 .addDef(FuncVReg)
6388 .addUse(ResTypeReg);
6389 MachineInstrBuilder MIB2 =
6390 BuildMI(BB, I, I.getDebugLoc(),
6391 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
6392 .addDef(ResVReg)
6393 .addUse(ResTypeReg)
6394 .addUse(FuncVReg);
6395 GR.add(ConstVal, MIB2);
6396 // mapping the function pointer to the used Function
6397 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun);
6398 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
6399 MIB1.constrainAllUses(TII, TRI, RBI);
6400 MIB2.constrainAllUses(TII, TRI, RBI);
6401 return true;
6402 }
6403 MachineInstrBuilder MIB3 =
6404 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
6405 .addDef(ResVReg)
6406 .addUse(GR.getSPIRVTypeID(ResType));
6407 GR.add(ConstVal, MIB3);
6408 MIB3.constrainAllUses(TII, TRI, RBI);
6409 return true;
6410 }
6411 assert(NewReg != ResVReg);
6412 return BuildCOPY(ResVReg, NewReg, I);
6413 }
6414 auto GlobalVar = cast<GlobalVariable>(GV);
6415 assert(GlobalVar->getName() != "llvm.global.annotations");
6416
6417 // Skip empty declaration for GVs with initializers till we get the decl with
6418 // passed initializer.
6419 if (hasInitializer(GlobalVar) && !Init)
6420 return true;
6421
6422 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
6423 getSpirvLinkageTypeFor(STI, *GV);
6424
6425 if (LnkType && *LnkType == SPIRV::LinkageType::Import)
6426 Init = nullptr;
6427
6428 const unsigned AddrSpace = GV->getAddressSpace();
6429 SPIRV::StorageClass::StorageClass StorageClass =
6430 addressSpaceToStorageClass(AddrSpace, STI);
6431 SPIRVTypeInst ResType =
6432 GR.getOrCreateSPIRVPointerType(GVType, I, StorageClass);
6433 Register Reg = GR.buildGlobalVariable(
6434 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
6435 GlobalVar->isConstant(), LnkType, MIRBuilder, true);
6436 // TODO: For AMDGCN, we pipe externally_initialized through via
6437 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
6438 // reverse translation. We should remove this once SPIR-V gains the ability to
6439 // express the concept.
6440 if (GlobalVar->isExternallyInitialized() &&
6441 STI.getTargetTriple().getVendor() == Triple::AMD) {
6442 constexpr unsigned ReadWriteINTEL = 3u;
6443 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL,
6444 {ReadWriteINTEL});
6445 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
6446 addStringImm(GV->getName(), MIB);
6447 }
6448 return Reg.isValid();
6449}
6450
6451bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
6452 SPIRVTypeInst ResType,
6453 MachineInstr &I) const {
6454 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
6455 return selectExtInst(ResVReg, ResType, I, CL::log10);
6456 }
6457
6458 // There is no log10 instruction in the GLSL Extended Instruction set, so it
6459 // is implemented as:
6460 // log10(x) = log2(x) * (1 / log2(10))
6461 // = log2(x) * 0.30103
6462
6463 MachineIRBuilder MIRBuilder(I);
6464 MachineBasicBlock &BB = *I.getParent();
6465
6466 // Build log2(x).
6467 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6468 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
6469 .addDef(VarReg)
6470 .addUse(GR.getSPIRVTypeID(ResType))
6471 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
6472 .addImm(GL::Log2)
6473 .add(I.getOperand(1))
6474 .constrainAllUses(TII, TRI, RBI);
6475
6476 // Build 0.30103.
6477 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
6478 ResType->getOpcode() == SPIRV::OpTypeFloat);
6479 // TODO: Add matrix implementation once supported by the HLSL frontend.
6480 SPIRVTypeInst SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector
6481 ? SPIRVTypeInst(GR.getSPIRVTypeForVReg(
6482 ResType->getOperand(1).getReg()))
6483 : ResType;
6484 Register ScaleReg =
6485 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
6486
6487 // Multiply log2(x) by 0.30103 to get log10(x) result.
6488 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
6489 ? SPIRV::OpVectorTimesScalar
6490 : SPIRV::OpFMulS;
6491 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
6492 .addDef(ResVReg)
6493 .addUse(GR.getSPIRVTypeID(ResType))
6494 .addUse(VarReg)
6495 .addUse(ScaleReg)
6496 .constrainAllUses(TII, TRI, RBI);
6497 return true;
6498}
6499
6500bool SPIRVInstructionSelector::selectFpowi(Register ResVReg,
6501 SPIRVTypeInst ResType,
6502 MachineInstr &I) const {
6503 // On OpenCL targets, pown(gentype x, intn n) maps directly.
6504 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std))
6505 return selectExtInst(ResVReg, ResType, I, CL::pown);
6506
6507 // On GLSL (Vulkan) targets, there is no integer-exponent power instruction.
6508 // Lower as: Pow(base, OpConvertSToF(exp)).
6509 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
6510 Register BaseReg = I.getOperand(1).getReg();
6511 Register ExpReg = I.getOperand(2).getReg();
6512 Register FloatExpReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6513 if (!selectOpWithSrcs(FloatExpReg, ResType, I, {ExpReg},
6514 SPIRV::OpConvertSToF))
6515 return false;
6516 return selectExtInst(ResVReg, ResType, I, GL::Pow,
6517 /*setMIFlags=*/true, /*useMISrc=*/false,
6518 {BaseReg, FloatExpReg});
6519 }
6520 return false;
6521}
6522
6523bool SPIRVInstructionSelector::selectModf(Register ResVReg,
6524 SPIRVTypeInst ResType,
6525 MachineInstr &I) const {
6526 // llvm.modf has a single arg --the number to be decomposed-- and returns a
6527 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
6528 // number to be decomposed and a pointer--, returns the fractional part and
6529 // the integral part is stored in the pointer argument. Therefore, we can't
6530 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
6531 // scaffolding to make it work. The idea is to create an alloca instruction
6532 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
6533 // from this ptr to place it in the struct. llvm.modf returns the fractional
6534 // part as the first element of the result, and the integral part as the
6535 // second element of the result.
6536
6537 // At this point, the return type is not a struct anymore, but rather two
6538 // independent elements of SPIRVResType. We can get each independent element
6539 // from I.getDefs() or I.getOperands().
6540 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
6541 MachineIRBuilder MIRBuilder(I);
6542 // Get pointer type for alloca variable.
6543 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
6544 ResType, MIRBuilder, SPIRV::StorageClass::Function);
6545 // Create new register for the pointer type of alloca variable.
6546 Register PtrTyReg =
6547 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
6548 MIRBuilder.getMRI()->setType(
6549 PtrTyReg,
6550 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
6551 GR.getPointerSize()));
6552
6553 // Assign SPIR-V type of the pointer type of the alloca variable to the
6554 // new register.
6555 GR.assignSPIRVTypeToVReg(PtrType, PtrTyReg, MIRBuilder.getMF());
6556 MachineBasicBlock &EntryBB = I.getMF()->front();
6557 MachineBasicBlock::iterator VarPos =
6558 getFirstValidInstructionInsertPoint(EntryBB);
6559 auto AllocaMIB =
6560 BuildMI(EntryBB, VarPos, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
6561 .addDef(PtrTyReg)
6562 .addUse(GR.getSPIRVTypeID(PtrType))
6563 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
6564 Register Variable = AllocaMIB->getOperand(0).getReg();
6565
6566 MachineBasicBlock &BB = *I.getParent();
6567 // Create the OpenCLLIB::modf instruction.
6568 auto MIB =
6569 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
6570 .addDef(ResVReg)
6571 .addUse(GR.getSPIRVTypeID(ResType))
6572 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
6573 .addImm(CL::modf)
6574 .setMIFlags(I.getFlags())
6575 .add(I.getOperand(I.getNumExplicitDefs())) // Floating point value.
6576 .addUse(Variable); // Pointer to integral part.
6577 // Assign the integral part stored in the ptr to the second element of the
6578 // result.
6579 Register IntegralPartReg = I.getOperand(1).getReg();
6580 if (IntegralPartReg.isValid()) {
6581 // Load the value from the pointer to integral part.
6582 auto LoadMIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
6583 .addDef(IntegralPartReg)
6584 .addUse(GR.getSPIRVTypeID(ResType))
6585 .addUse(Variable);
6586 LoadMIB.constrainAllUses(TII, TRI, RBI);
6587 return true;
6588 }
6589
6590 MIB.constrainAllUses(TII, TRI, RBI);
6591 return true;
6592 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
6593 assert(false && "GLSL::Modf is deprecated.");
6594 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
6595 return false;
6596 }
6597 return false;
6598}
6599
6600// Generate the instructions to load 3-element vector builtin input
6601// IDs/Indices.
6602// Like: GlobalInvocationId, LocalInvocationId, etc....
6603
6604bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
6605 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
6606 SPIRVTypeInst ResType, MachineInstr &I) const {
6607 MachineIRBuilder MIRBuilder(I);
6608 const SPIRVTypeInst Vec3Ty =
6609 GR.getOrCreateSPIRVVectorType(ResType, 3, MIRBuilder, false);
6610 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
6611 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
6612
6613 // Create new register for the input ID builtin variable.
6614 Register NewRegister =
6615 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
6616 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64));
6617 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
6618
6619 // Build global variable with the necessary decorations for the input ID
6620 // builtin variable.
6621 Register Variable = GR.buildGlobalVariable(
6622 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
6623 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
6624 false);
6625
6626 // Create new register for loading value.
6627 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
6628 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6629 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64));
6630 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
6631
6632 // Load v3uint value from the global variable.
6633 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
6634 .addDef(LoadedRegister)
6635 .addUse(GR.getSPIRVTypeID(Vec3Ty))
6636 .addUse(Variable);
6637
6638 // Get the input ID index. Expecting operand is a constant immediate value,
6639 // wrapped in a type assignment.
6640 assert(I.getOperand(2).isReg());
6641 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI);
6642
6643 // Extract the input ID from the loaded vector value.
6644 MachineBasicBlock &BB = *I.getParent();
6645 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
6646 .addDef(ResVReg)
6647 .addUse(GR.getSPIRVTypeID(ResType))
6648 .addUse(LoadedRegister)
6649 .addImm(ThreadId);
6650 MIB.constrainAllUses(TII, TRI, RBI);
6651 return true;
6652}
6653
6654// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
6655// Like LocalInvocationIndex
6656bool SPIRVInstructionSelector::loadBuiltinInputID(
6657 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
6658 SPIRVTypeInst ResType, MachineInstr &I) const {
6659 MachineIRBuilder MIRBuilder(I);
6660 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
6661 ResType, MIRBuilder, SPIRV::StorageClass::Input);
6662
6663 // Create new register for the input ID builtin variable.
6664 Register NewRegister =
6665 MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType));
6666 MIRBuilder.getMRI()->setType(
6667 NewRegister,
6668 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input),
6669 GR.getPointerSize()));
6670 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
6671
6672 // Build global variable with the necessary decorations for the input ID
6673 // builtin variable.
6674 Register Variable = GR.buildGlobalVariable(
6675 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
6676 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
6677 false);
6678
6679 // Load uint value from the global variable.
6680 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
6681 .addDef(ResVReg)
6682 .addUse(GR.getSPIRVTypeID(ResType))
6683 .addUse(Variable);
6684
6685 MIB.constrainAllUses(TII, TRI, RBI);
6686 return true;
6687}
6688
6689SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type,
6690 MachineInstr &I) const {
6691 MachineIRBuilder MIRBuilder(I);
6692 if (Type->getOpcode() != SPIRV::OpTypeVector)
6693 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder, false);
6694
6695 uint64_t VectorSize = Type->getOperand(2).getImm();
6696 if (VectorSize == 4)
6697 return Type;
6698
6699 Register ScalarTypeReg = Type->getOperand(1).getReg();
6700 const SPIRVTypeInst ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg);
6701 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder, false);
6702}
6703
6704bool SPIRVInstructionSelector::loadHandleBeforePosition(
6705 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
6706 MachineInstr &Pos) const {
6707
6708 assert(HandleDef.getIntrinsicID() ==
6709 Intrinsic::spv_resource_handlefrombinding);
6710 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI);
6711 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI);
6712 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI);
6713 Register IndexReg = HandleDef.getOperand(5).getReg();
6714 std::string Name =
6715 getStringValueFromReg(HandleDef.getOperand(6).getReg(), *MRI);
6716
6717 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
6718 MachineIRBuilder MIRBuilder(HandleDef);
6719 SPIRVTypeInst VarType = ResType;
6720 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
6721
6722 if (IsStructuredBuffer) {
6723 VarType = GR.getPointeeType(ResType);
6724 SC = GR.getPointerStorageClass(ResType);
6725 }
6726
6727 if (ResType->getOpcode() == SPIRV::OpTypeImage && ArraySize == 0)
6728 MIRBuilder.buildInstr(SPIRV::OpCapability)
6729 .addImm(SPIRV::Capability::RuntimeDescriptorArrayEXT);
6730
6731 Register VarReg =
6732 buildPointerToResource(SPIRVTypeInst(VarType), SC, Set, Binding,
6733 ArraySize, IndexReg, Name, MIRBuilder);
6734
6735 // The handle for the buffer is the pointer to the resource. For an image, the
6736 // handle is the image object. So images get an extra load.
6737 uint32_t LoadOpcode =
6738 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
6739 GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF());
6740 BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), TII.get(LoadOpcode))
6741 .addDef(HandleReg)
6742 .addUse(GR.getSPIRVTypeID(ResType))
6743 .addUse(VarReg)
6744 .constrainAllUses(TII, TRI, RBI);
6745 return true;
6746}
6747
6748void SPIRVInstructionSelector::errorIfInstrOutsideShader(
6749 MachineInstr &I) const {
6750 if (!STI.isShader()) {
6751 std::string DiagMsg;
6752 raw_string_ostream OS(DiagMsg);
6753 I.print(OS, true, false, false, false);
6754 DiagMsg += " is only supported in shaders.\n";
6755 report_fatal_error(DiagMsg.c_str(), false);
6756 }
6757}
6758
6759namespace llvm {
6760InstructionSelector *
6761createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
6762 const SPIRVSubtarget &Subtarget,
6763 const RegisterBankInfo &RBI) {
6764 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
6765}
6766} // namespace llvm
UseMI
MachineInstrBuilder & UseMI
Definition AArch64ExpandPseudoInsts.cpp:127
GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_PREDICATES_INIT
GET_GLOBALISEL_TEMPORARIES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
Generic
@ Generic
Definition AArch64MCAsmInfo.cpp:24
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
selectUnmergeValues
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Definition ARMInstructionSelector.cpp:264
MBB
MachineBasicBlock & MBB
Definition ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
SwapBits
static uint8_t SwapBits(uint8_t Val)
Definition BPFMCCodeEmitter.cpp:122
true
basic Basic Alias true
Definition BasicAliasAnalysis.cpp:2057
X
#define X(NUM, ENUM, NAME)
Definition ELF.h:851
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Binding
DXIL Resource Implicit Binding
Definition DXILResourceImplicitBinding.cpp:185
DEBUG_TYPE
#define DEBUG_TYPE
Definition GenericCycleImpl.h:31
GenericMachineInstrs.h
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
TII
const HexagonInstrInfo * TII
Definition HexagonCopyToCombine.cpp:118
MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:110
LLVMIntType
LLVMTypeRef LLVMIntType(unsigned NumBits)
Definition Core.cpp:729
NumOps
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
Definition ItaniumDemangle.h:3452
Direction
Loop::LoopBounds::Direction Direction
Definition LoopInfo.cpp:253
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
Reg
Register Reg
Definition MachineSink.cpp:2126
TRI
Register const TargetRegisterInfo * TRI
Definition MachineSink.cpp:2127
Register
Promote Memory to Register
Definition Mem2Reg.cpp:110
T1
#define T1
Definition Mips16ISelLowering.cpp:281
OpIdx
MachineInstr unsigned OpIdx
Definition NVPTXPrologEpilogPass.cpp:56
High
uint64_t High
Definition NVVMIntrRange.cpp:46
II
uint64_t IntrinsicInst * II
Definition NVVMIntrRange.cpp:46
getName
static StringRef getName(Value *V)
Definition ProvenanceAnalysisEvaluator.cpp:20
getFCmpOpcode
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
Definition RISCVInstructionSelector.cpp:1825
getOneFP
static APFloat getOneFP(const Type *LLVMFloatTy)
Definition SPIRVInstructionSelector.cpp:2763
isUSMStorageClass
static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVInstructionSelector.cpp:2495
isASCastInGVar
static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg)
Definition SPIRVInstructionSelector.cpp:2506
mayApplyGenericSelection
static bool mayApplyGenericSelection(unsigned Opcode)
Definition SPIRVInstructionSelector.cpp:948
getZeroFP
static APFloat getZeroFP(const Type *LLVMFloatTy)
Definition SPIRVInstructionSelector.cpp:2749
ExtInstList
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
Definition SPIRVInstructionSelector.cpp:45
intrinsicHasSideEffects
static bool intrinsicHasSideEffects(Intrinsic::ID ID)
Definition SPIRVInstructionSelector.cpp:662
getBoolCmpOpcode
static unsigned getBoolCmpOpcode(unsigned PredNum)
Definition SPIRVInstructionSelector.cpp:2737
getICmpOpcode
static unsigned getICmpOpcode(unsigned PredNum)
Definition SPIRVInstructionSelector.cpp:2697
isOpcodeWithNoSideEffects
static bool isOpcodeWithNoSideEffects(unsigned Opcode)
Definition SPIRVInstructionSelector.cpp:728
addMemoryOperands
static void addMemoryOperands(MachineMemOperand *MemOp, MachineInstrBuilder &MIB, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR)
Definition SPIRVInstructionSelector.cpp:1889
isConstReg
static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef)
Definition SPIRVInstructionSelector.cpp:594
getPtrCmpOpcode
static unsigned getPtrCmpOpcode(unsigned Pred)
Definition SPIRVInstructionSelector.cpp:2725
getVectorSizeOrOne
unsigned getVectorSizeOrOne(SPIRVTypeInst Type)
Definition SPIRVInstructionSelector.cpp:3236
isDead
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition SPIRVInstructionSelector.cpp:762
SPIRV
spirv structurize SPIRV
Definition SPIRVStructurizer.cpp:1218
BaseType
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
Definition SafepointIRVerifier.cpp:311
StringExtras.h
This file contains some functions that are useful when dealing with strings.
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:114
Y
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
Mul
BinaryOperator * Mul
Definition X86PartialReduction.cpp:75
llvm::APFloatBase::IEEEsingle
static const fltSemantics & IEEEsingle()
Definition APFloat.h:296
llvm::APFloatBase::IEEEdouble
static const fltSemantics & IEEEdouble()
Definition APFloat.h:297
llvm::APFloatBase::IEEEhalf
static const fltSemantics & IEEEhalf()
Definition APFloat.h:294
llvm::APFloat::getOne
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1143
llvm::APFloat::getZero
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1134
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:235
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition BlockFrequencyInfo.h:38
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:679
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:705
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:706
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:682
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:691
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:680
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:681
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:700
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:699
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:703
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:690
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:684
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:687
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:688
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:683
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:685
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:698
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:704
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:692
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:702
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:689
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:686
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:390
llvm::DenseMapBase::size
unsigned size() const
Definition DenseMap.h:110
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:358
llvm::GIntrinsic
Represents a call to an intrinsic.
Definition GenericMachineInstrs.h:529
llvm::GIntrinsic::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Definition GenericMachineInstrs.h:531
llvm::GlobalValue::getAddressSpace
unsigned getAddressSpace() const
Definition GlobalValue.h:207
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:663
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
llvm::IntegerType::get
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:354
llvm::LLT::scalar
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition LowLevelType.h:88
llvm::LLT::isValid
constexpr bool isValid() const
Definition LowLevelType.h:262
llvm::LLT::getNumElements
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition LowLevelType.h:318
llvm::LLT::isVector
constexpr bool isVector() const
Definition LowLevelType.h:286
llvm::LLT::pointer
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition LowLevelType.h:115
llvm::LLT::fixed_vector
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition LowLevelType.h:203
llvm::MachineBasicBlock::getNumber
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
Definition MachineBasicBlock.h:1274
llvm::MachineBasicBlock::getFirstNonPHI
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition MachineBasicBlock.cpp:202
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition MachineBasicBlock.h:330
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition MachineBasicBlock.h:348
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition MachineFunction.h:804
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:755
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition MachineIRBuilder.h:237
llvm::MachineIRBuilder::getInsertPt
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
Definition MachineIRBuilder.h:335
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition MachineIRBuilder.h:423
llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition MachineIRBuilder.h:289
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition MachineIRBuilder.h:311
llvm::MachineInstrBuilder::constrainAllUses
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
Definition MachineInstrBuilder.h:441
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
Definition MachineInstrBuilder.h:225
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Definition MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition MachineInstrBuilder.h:233
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition MachineInstrBuilder.h:326
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition MachineInstrBuilder.h:248
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition MachineInstrBuilder.h:218
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition MachineInstrBuilder.h:379
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition MachineInstrBuilder.h:191
llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:73
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition MachineInstr.h:601
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition MachineInstr.h:373
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition MachineInstr.h:604
llvm::MachineInstr::getNumExplicitOperands
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
Definition MachineInstr.cpp:851
llvm::MachineInstr::getNumExplicitDefs
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
Definition MachineInstr.cpp:869
llvm::MachineInstr::emitGenericError
LLVM_ABI void emitGenericError(const Twine &ErrMsg) const
Definition MachineInstr.cpp:2373
llvm::MachineInstr::getMF
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
Definition MachineInstr.cpp:786
llvm::MachineInstr::getDebugLoc
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition MachineInstr.h:525
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition MachineInstr.h:609
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition MachineMemOperand.h:130
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition MachineMemOperand.h:141
llvm::MachineMemOperand::MONonTemporal
@ MONonTemporal
The memory access is non-temporal.
Definition MachineMemOperand.h:143
llvm::MachineOperand::getImm
int64_t getImm() const
Definition MachineOperand.h:560
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition MachineOperand.h:331
llvm::MachineOperand::getMBB
MachineBasicBlock * getMBB() const
Definition MachineOperand.h:575
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition MachineOperand.h:372
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:53
llvm::MachineRegisterInfo::use_instr_iterator
defusechain_instr_iterator< true, false, false, true > use_instr_iterator
use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the specified register,...
Definition MachineRegisterInfo.h:478
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition MachineRegisterInfo.h:648
llvm::MachineRegisterInfo::getVRegDef
LLVM_ABI MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition MachineRegisterInfo.cpp:404
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition MachineRegisterInfo.h:480
llvm::MachineRegisterInfo::def_instr_end
static def_instr_iterator def_instr_end()
Definition MachineRegisterInfo.h:404
llvm::MachineRegisterInfo::def_instr_iterator
defusechain_instr_iterator< false, true, false, true > def_instr_iterator
def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the specified register,...
Definition MachineRegisterInfo.h:399
llvm::MachineRegisterInfo::createVirtualRegister
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition MachineRegisterInfo.cpp:154
llvm::MachineRegisterInfo::def_instr_begin
def_instr_iterator def_instr_begin(Register RegNo) const
Definition MachineRegisterInfo.h:401
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition MachineRegisterInfo.h:771
llvm::MachineRegisterInfo::use_instr_end
static use_instr_iterator use_instr_end()
Definition MachineRegisterInfo.h:483
llvm::MachineRegisterInfo::use_nodbg_instructions
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
Definition MachineRegisterInfo.h:546
llvm::MachineRegisterInfo::setType
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
Definition MachineRegisterInfo.cpp:185
llvm::MachineRegisterInfo::getMF
const MachineFunction & getMF() const
Definition MachineRegisterInfo.h:189
llvm::MachineRegisterInfo::setRegClass
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition MachineRegisterInfo.cpp:58
llvm::MachineRegisterInfo::createGenericVirtualRegister
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition MachineRegisterInfo.cpp:191
llvm::MachineRegisterInfo::getRegClassOrNull
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
Definition MachineRegisterInfo.h:665
llvm::MachineRegisterInfo::use_instructions
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Definition MachineRegisterInfo.h:488
llvm::MachineRegisterInfo::getNumVirtRegs
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
Definition MachineRegisterInfo.h:796
llvm::MachineRegisterInfo::replaceRegWith
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Definition MachineRegisterInfo.cpp:386
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition ProfileSummaryInfo.h:42
llvm::RegisterBankInfo
Holds all the information related to register banks.
Definition RegisterBankInfo.h:40
llvm::Register
Wrapper class representing virtual and physical registers.
Definition Register.h:20
llvm::Register::isValid
constexpr bool isValid() const
Definition Register.h:112
llvm::Register::isPhysical
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
llvm::SPIRVGlobalRegistry::isScalarOrVectorSigned
bool isScalarOrVectorSigned(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1484
llvm::SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage
SPIRVTypeInst getOrCreateOpTypeSampledImage(SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:1732
llvm::SPIRVGlobalRegistry::assignSPIRVTypeToVReg
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
Definition SPIRVGlobalRegistry.cpp:130
llvm::SPIRVGlobalRegistry::getRegClass
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
Definition SPIRVGlobalRegistry.cpp:2094
llvm::SPIRVGlobalRegistry::getOrAddMemAliasingINTELInst
MachineInstr * getOrAddMemAliasingINTELInst(MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD)
Definition SPIRVGlobalRegistry.cpp:2159
llvm::SPIRVGlobalRegistry::isAggregateType
bool isAggregateType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.h:371
llvm::SPIRVGlobalRegistry::getScalarOrVectorBitWidth
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1449
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:1862
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVVectorType
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
Definition SPIRVGlobalRegistry.cpp:1961
llvm::SPIRVGlobalRegistry::buildGlobalVariable
Register buildGlobalVariable(Register Reg, SPIRVTypeInst BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
Definition SPIRVGlobalRegistry.cpp:805
llvm::SPIRVGlobalRegistry::getResultType
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
Definition SPIRVGlobalRegistry.cpp:1332
llvm::SPIRVGlobalRegistry::getScalarOrVectorComponentCount
unsigned getScalarOrVectorComponentCount(Register VReg) const
Definition SPIRVGlobalRegistry.cpp:1422
llvm::SPIRVGlobalRegistry::getTypeForSPIRVType
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
Definition SPIRVGlobalRegistry.h:324
llvm::SPIRVGlobalRegistry::isBitcastCompatible
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
Definition SPIRVGlobalRegistry.cpp:1500
llvm::SPIRVGlobalRegistry::getOrCreateConstFP
Register getOrCreateConstFP(APFloat Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:359
llvm::SPIRVGlobalRegistry::getRegType
LLT getRegType(SPIRVTypeInst SpvType) const
Definition SPIRVGlobalRegistry.cpp:2121
llvm::SPIRVGlobalRegistry::invalidateMachineInstr
void invalidateMachineInstr(MachineInstr *MI)
Definition SPIRVGlobalRegistry.cpp:233
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVBoolType
SPIRVTypeInst getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
Definition SPIRVGlobalRegistry.cpp:1936
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVPointerType
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC)
Definition SPIRVGlobalRegistry.cpp:2002
llvm::SPIRVGlobalRegistry::isScalarOfType
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Definition SPIRVGlobalRegistry.cpp:1387
llvm::SPIRVGlobalRegistry::getSPIRVTypeID
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
Definition SPIRVGlobalRegistry.cpp:1159
llvm::SPIRVGlobalRegistry::getOrCreateConstInt
Register getOrCreateConstInt(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:412
llvm::SPIRVGlobalRegistry::getOrCreateConstIntArray
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:671
llvm::SPIRVGlobalRegistry::findValueAttrs
bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name)
Definition SPIRVGlobalRegistry.h:176
llvm::SPIRVGlobalRegistry::retrieveScalarOrVectorIntType
SPIRVTypeInst retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1478
llvm::SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding
Register getOrCreateGlobalVariableWithBinding(SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:917
llvm::SPIRVGlobalRegistry::changePointerStorageClass
SPIRVTypeInst changePointerStorageClass(SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I)
Definition SPIRVGlobalRegistry.cpp:2013
llvm::SPIRVGlobalRegistry::getOrCreateConstVector
Register getOrCreateConstVector(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:622
llvm::SPIRVGlobalRegistry::buildConstantFP
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType=nullptr)
Definition SPIRVGlobalRegistry.cpp:526
llvm::SPIRVGlobalRegistry::addGlobalObject
void addGlobalObject(const Value *V, const MachineFunction *MF, Register R)
Definition SPIRVGlobalRegistry.h:122
llvm::SPIRVGlobalRegistry::getScalarOrVectorComponentType
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1436
llvm::SPIRVGlobalRegistry::recordFunctionPointer
void recordFunctionPointer(const MachineOperand *MO, const Function *F)
Definition SPIRVGlobalRegistry.h:259
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVFloatType
SPIRVTypeInst getOrCreateSPIRVFloatType(unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:1915
llvm::SPIRVGlobalRegistry::getPointeeType
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
Definition SPIRVGlobalRegistry.cpp:1489
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVType
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
Definition SPIRVGlobalRegistry.h:310
llvm::SPIRVGlobalRegistry::isScalarOrVectorOfType
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
Definition SPIRVGlobalRegistry.cpp:1394
llvm::SPIRVGlobalRegistry::setCurrentFunc
MachineFunction * setCurrentFunc(MachineFunction &MF)
Definition SPIRVGlobalRegistry.h:364
llvm::SPIRVGlobalRegistry::getOrCreateConstNullPtr
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType)
Definition SPIRVGlobalRegistry.cpp:754
llvm::SPIRVGlobalRegistry::getSPIRVTypeForVReg
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
Definition SPIRVGlobalRegistry.cpp:1321
llvm::SPIRVGlobalRegistry::getDeducedGlobalValueType
Type * getDeducedGlobalValueType(const GlobalValue *Global)
Definition SPIRVGlobalRegistry.h:212
llvm::SPIRVGlobalRegistry::getOrCreateUndef
Register getOrCreateUndef(MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:2060
llvm::SPIRVGlobalRegistry::getPointerStorageClass
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
Definition SPIRVGlobalRegistry.cpp:1520
llvm::SPIRVIRMapping::erase
bool erase(const MachineInstr *MI)
Definition SPIRVIRMapping.h:211
llvm::SPIRVIRMapping::add
bool add(SPIRV::IRHandle Handle, const MachineInstr *MI)
Definition SPIRVIRMapping.h:189
llvm::SPIRVIRMapping::find
Register find(SPIRV::IRHandle Handle, const MachineFunction *MF)
Definition SPIRVIRMapping.h:235
llvm::SPIRVSubtarget::isPhysicalSPIRV
bool isPhysicalSPIRV() const
Definition SPIRVSubtarget.h:92
llvm::SPIRVSubtarget::isAtLeastSPIRVVer
bool isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const
Definition SPIRVSubtarget.cpp:143
llvm::SPIRVSubtarget::canUseExtInstSet
bool canUseExtInstSet(SPIRV::InstructionSet::InstructionSet E) const
Definition SPIRVSubtarget.cpp:130
llvm::SPIRVSubtarget::isLogicalSPIRV
bool isLogicalSPIRV() const
Definition SPIRVSubtarget.h:89
llvm::SPIRVSubtarget::canUseExtension
bool canUseExtension(SPIRV::Extension::Extension E) const
Definition SPIRVSubtarget.cpp:126
llvm::SmallPtrSetImpl::erase
bool erase(PtrType Ptr)
Remove pointer from the set.
Definition SmallPtrSet.h:404
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:389
llvm::SmallPtrSetImpl::contains
bool contains(ConstPtrType Ptr) const
Definition SmallPtrSet.h:461
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:527
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:964
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition SmallVector.h:669
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:422
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1223
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition StringRef.h:143
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:483
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::HalfTyID
@ HalfTyID
16-bit floating point type
Definition Type.h:57
llvm::Type::FloatTyID
@ FloatTyID
32-bit floating point type
Definition Type.h:59
llvm::Type::DoubleTyID
@ DoubleTyID
64-bit floating point type
Definition Type.h:60
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:278
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition Type.h:321
llvm::Type::getTypeID
TypeID getTypeID() const
Return the type id for the type.
Definition Type.h:138
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:318
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
uint32_t
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPU::HSAMD::Kernel::Arg::Key::IsConst
constexpr char IsConst[]
Key for Kernel::Arg::Metadata::mIsConst.
Definition AMDGPUMetadata.h:197
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition BitmaskEnum.h:126
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
llvm::dwarf_linker::DebugSectionKind::DebugLoc
@ DebugLoc
Definition DWARFLinkerBase.h:34
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm::rdf::Def
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
llvm::rdf::Instr
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
llvm::rdf::Use
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
llvm::sframe::BaseReg
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::buildOpName
void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)
Definition SPIRVUtils.cpp:187
llvm::ThreadPriority::Low
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
llvm::Offset
@ Offset
Definition DWP.cpp:532
llvm::Value
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1739
llvm::getIConstValSext
int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:464
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition MachineInstrBuilder.h:449
llvm::isTypeFoldingSupported
bool isTypeFoldingSupported(unsigned Opcode)
Definition SPIRVUtils.cpp:1066
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::Int32Ty
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
Definition InstrProf.h:328
llvm::addNumImm
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
Definition SPIRVUtils.cpp:156
llvm::salvageDebugInfo
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1666
llvm::constrainSelectedInstRegOperands
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:155
llvm::isPreISelGenericOpcode
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
Definition TargetOpcodes.h:30
llvm::createVirtualRegister
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
Definition SPIRVUtils.cpp:892
llvm::getArrayComponentCount
unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)
Definition SPIRVUtils.cpp:1103
llvm::getIConstVal
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:458
llvm::createContinuedInstructions
SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)
Definition SPIRVUtils.cpp:957
llvm::getMemSemanticsForStorageClass
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.cpp:373
llvm::storageClassToAddressSpace
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:248
llvm::getFirstValidInstructionInsertPoint
MachineBasicBlock::iterator getFirstValidInstructionInsertPoint(MachineBasicBlock &BB)
Definition SPIRVUtils.cpp:1109
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1746
llvm::buildOpDecorate
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)
Definition SPIRVUtils.cpp:214
llvm::getOpVariableMBBIt
MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I)
Definition SPIRVUtils.cpp:299
llvm::getImm
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1084
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:279
llvm::toTypedPointer
Type * toTypedPointer(Type *Ty)
Definition SPIRVUtils.h:466
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isGenericCastablePtr
constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:233
llvm::SmallVector
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Definition SmallVector.h:1149
llvm::passCopy
MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1071
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::getSpirvLinkageTypeFor
std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)
Definition SPIRVUtils.cpp:1201
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:904
llvm::addressSpaceToStorageClass
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
Definition SPIRVUtils.cpp:333
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::getMemScope
SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id)
Definition SPIRVUtils.cpp:409
llvm::createSPIRVInstructionSelector
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
Definition SPIRVInstructionSelector.cpp:6761
llvm::getStringValueFromReg
std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)
Definition SPIRVUtils.cpp:146
llvm::foldImm
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1093
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:23
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getDefInstrMaybeConstant
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:435
llvm::BitWidth
constexpr unsigned BitWidth
Definition BitmaskEnum.h:219
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::hasInitializer
bool hasInitializer(const GlobalVariable *GV)
Definition SPIRVUtils.h:350
llvm::isSpvIntrinsic
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
Definition SPIRVUtils.cpp:470
llvm::addStringImm
void addStringImm(const StringRef &Str, MCInst &Inst)
Definition SPIRVUtils.cpp:117
llvm::getVRegDef
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
Definition SPIRVUtils.cpp:843
llvm::getMemSemantics
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition SPIRVUtils.cpp:391
llvm::getLinkStringForBuiltIn
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
Definition SPIRVBaseInfo.cpp:202
llvm::isTriviallyDead
LLVM_ABI bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
Definition Utils.cpp:220
N
#define N
Generated on for LLVM by doxygen 1.14.0