LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("addqv")) {
1051 // 'aarch64.sve.addqv'.
1052 if (!F->getReturnType()->isFPOrFPVectorTy())
1053 return false;
1054
1055 auto Args = F->getFunctionType()->params();
1056 Type *Tys[] = {F->getReturnType(), Args[1]};
1057 NewFn = Intrinsic::getOrInsertDeclaration(
1058 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1059 return true;
1060 }
1061
1062 if (Name.consume_front("ld")) {
1063 // 'aarch64.sve.ld*'.
1064 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1065 if (LdRegex.match(Name)) {
1066 Type *ScalarTy =
1067 cast<VectorType>(F->getReturnType())->getElementType();
1068 ElementCount EC =
1069 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1070 assert(F->arg_size() == 2 &&
1071 "Expected 2 arguments for ld* intrinsic.");
1072 Type *PtrTy = F->getArg(1)->getType();
1073 Type *Ty = VectorType::get(ScalarTy, EC);
1074 static const Intrinsic::ID LoadIDs[] = {
1075 Intrinsic::aarch64_sve_ld2_sret,
1076 Intrinsic::aarch64_sve_ld3_sret,
1077 Intrinsic::aarch64_sve_ld4_sret,
1078 };
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1081 return true;
1082 }
1083 return false; // No other 'aarch64.sve.ld*'.
1084 }
1085
1086 if (Name.consume_front("tuple.")) {
1087 // 'aarch64.sve.tuple.*'.
1088 if (Name.starts_with("get")) {
1089 // 'aarch64.sve.tuple.get*'.
1090 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1091 NewFn = Intrinsic::getOrInsertDeclaration(
1092 F->getParent(), Intrinsic::vector_extract, Tys);
1093 return true;
1094 }
1095
1096 if (Name.starts_with("set")) {
1097 // 'aarch64.sve.tuple.set*'.
1098 auto Args = F->getFunctionType()->params();
1099 Type *Tys[] = {Args[0], Args[2], Args[1]};
1100 NewFn = Intrinsic::getOrInsertDeclaration(
1101 F->getParent(), Intrinsic::vector_insert, Tys);
1102 return true;
1103 }
1104
1105 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1106 if (CreateTupleRegex.match(Name)) {
1107 // 'aarch64.sve.tuple.create*'.
1108 auto Args = F->getFunctionType()->params();
1109 Type *Tys[] = {F->getReturnType(), Args[1]};
1110 NewFn = Intrinsic::getOrInsertDeclaration(
1111 F->getParent(), Intrinsic::vector_insert, Tys);
1112 return true;
1113 }
1114 return false; // No other 'aarch64.sve.tuple.*'.
1115 }
1116
1117 if (Name.starts_with("rev.nxv")) {
1118 // 'aarch64.sve.rev.<Ty>'
1119 NewFn = Intrinsic::getOrInsertDeclaration(
1120 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1121 return true;
1122 }
1123
1124 return false; // No other 'aarch64.sve.*'.
1125 }
1126 if (Name.consume_front("sme.")) {
1127 // 'aarch64.sme.*'.
1128 if (Name.consume_front("ftmopa.")) {
1129 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1130 // intrinsics to model their FPMR dependency.
1131 Intrinsic::ID ID =
1132 StringSwitch<Intrinsic::ID>(Name)
1133 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1134 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1135 .Default(Intrinsic::not_intrinsic);
1136 if (ID != Intrinsic::not_intrinsic) {
1137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1138 return true;
1139 }
1140 return false; // No other 'aarch64.sme.ftmopa.*'.
1141 }
1142
1143 return false; // No other 'aarch64.sme.*'.
1144 }
1145 }
1146 return false; // No other 'arm.*', 'aarch64.*'.
1147}
1148
1149static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1150 StringRef Name) {
1151 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1152 Intrinsic::ID ID =
1153 StringSwitch<Intrinsic::ID>(Name)
1154 .Case("im2col.3d",
1155 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1156 .Case("im2col.4d",
1157 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1158 .Case("im2col.5d",
1159 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1160 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1161 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1162 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1163 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1164 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1165 .Default(Intrinsic::not_intrinsic);
1166
1167 if (ID == Intrinsic::not_intrinsic)
1168 return ID;
1169
1170 // These intrinsics may need upgrade for two reasons:
1171 // (1) When the address-space of the first argument is shared[AS=3]
1172 // (and we upgrade it to use shared_cluster address-space[AS=7])
1173 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1174 NVPTXAS::ADDRESS_SPACE_SHARED)
1175 return ID;
1176
1177 // (2) When there are only two boolean flag arguments at the end:
1178 //
1179 // The last three parameters of the older version of these
1180 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1181 //
1182 // The newer version reads as:
1183 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1184 //
1185 // So, when the type of the [N-3]rd argument is "not i1", then
1186 // it is the older version and we need to upgrade.
1187 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1188 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1189 if (!ArgType->isIntegerTy(1))
1190 return ID;
1191 }
1192
1193 return Intrinsic::not_intrinsic;
1194}
1195
1196// The legacy TMA reduction intrinsics encode the reduction operator in their
1197// name, while the current ones take it as an immediate argument. Map the
1198// operator part of a legacy name to the corresponding immediate value.
1199static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1200 return StringSwitch<std::optional<unsigned>>(Name)
1201 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1202 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1203 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1204 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1205 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1206 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1207 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1208 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1209 .Default(std::nullopt);
1210}
1211
1212static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1213 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1214 return Intrinsic::not_intrinsic;
1215
1216 auto [RedOpName, ShapeName] = Name.split('.');
1217 if (!getNVPTXTMAReductionOp(RedOpName))
1218 return Intrinsic::not_intrinsic;
1219
1220 return StringSwitch<Intrinsic::ID>(ShapeName)
1221 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1222 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1223 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1224 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1225 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1226 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1227 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1228 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1229 .Default(Intrinsic::not_intrinsic);
1230}
1231
1232static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1233 StringRef Name) {
1234 if (Name.consume_front("mapa.shared.cluster"))
1235 if (F->getReturnType()->getPointerAddressSpace() ==
1236 NVPTXAS::ADDRESS_SPACE_SHARED)
1237 return Intrinsic::nvvm_mapa_shared_cluster;
1238
1239 if (Name.consume_front("cp.async.bulk.")) {
1240 Intrinsic::ID ID =
1241 StringSwitch<Intrinsic::ID>(Name)
1242 .Case("global.to.shared.cluster",
1243 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1244 .Case("shared.cta.to.cluster",
1245 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1246 .Default(Intrinsic::not_intrinsic);
1247
1248 if (ID != Intrinsic::not_intrinsic)
1249 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1250 NVPTXAS::ADDRESS_SPACE_SHARED)
1251 return ID;
1252 }
1253
1254 return Intrinsic::not_intrinsic;
1255}
1256
1257static Intrinsic::ID
1258shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1259 if (!Name.consume_front("tcgen05.commit."))
1260 return Intrinsic::not_intrinsic;
1261
1262 if (Name.consume_front("shared."))
1263 return StringSwitch<Intrinsic::ID>(Name)
1264 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1265 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1266 .Default(Intrinsic::not_intrinsic);
1267
1268 if (Name.consume_front("mc.shared.")) {
1269 // Only upgrade older i16 mc variants.
1270 if (!F->getArg(1)->getType()->isIntegerTy(16))
1271 return Intrinsic::not_intrinsic;
1272
1273 return StringSwitch<Intrinsic::ID>(Name)
1274 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1275 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1276 .Default(Intrinsic::not_intrinsic);
1277 }
1278
1279 return Intrinsic::not_intrinsic;
1280}
1281
1282static Intrinsic::ID
1283shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1284 if (F->arg_size() != 2)
1285 return Intrinsic::not_intrinsic;
1286
1287 if (Name.consume_front("tcgen05.alloc.shared.") ||
1288 Name.consume_front("tcgen05.alloc."))
1289 return StringSwitch<Intrinsic::ID>(Name)
1290 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1291 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1292 .Default(Intrinsic::not_intrinsic);
1293
1294 if (Name.consume_front("tcgen05.dealloc."))
1295 return StringSwitch<Intrinsic::ID>(Name)
1296 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1297 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1298 .Default(Intrinsic::not_intrinsic);
1299
1300 return Intrinsic::not_intrinsic;
1301}
1302
1303static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1304 if (Name.consume_front("fma.rn."))
1305 return StringSwitch<Intrinsic::ID>(Name)
1306 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1307 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1308 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1309 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1310 .Default(Intrinsic::not_intrinsic);
1311
1312 if (Name.consume_front("fmax."))
1313 return StringSwitch<Intrinsic::ID>(Name)
1314 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1315 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1316 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1317 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1318 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1319 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1320 .Case("ftz.nan.xorsign.abs.bf16",
1321 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1322 .Case("ftz.nan.xorsign.abs.bf16x2",
1323 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1324 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1325 .Case("ftz.xorsign.abs.bf16x2",
1326 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1327 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1328 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1329 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1330 .Case("nan.xorsign.abs.bf16x2",
1331 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1332 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1333 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1334 .Default(Intrinsic::not_intrinsic);
1335
1336 if (Name.consume_front("fmin."))
1337 return StringSwitch<Intrinsic::ID>(Name)
1338 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1339 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1340 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1341 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1342 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1343 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1344 .Case("ftz.nan.xorsign.abs.bf16",
1345 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1346 .Case("ftz.nan.xorsign.abs.bf16x2",
1347 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1348 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1349 .Case("ftz.xorsign.abs.bf16x2",
1350 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1351 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1352 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1353 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1354 .Case("nan.xorsign.abs.bf16x2",
1355 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1356 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1357 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1358 .Default(Intrinsic::not_intrinsic);
1359
1360 if (Name.consume_front("neg."))
1361 return StringSwitch<Intrinsic::ID>(Name)
1362 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1363 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1364 .Default(Intrinsic::not_intrinsic);
1365
1366 return Intrinsic::not_intrinsic;
1367}
1368
1369static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1370 StringRef Name) {
1371 if (!Name.consume_front("tcgen05.mma."))
1372 return Intrinsic::not_intrinsic;
1373
1374 // tcgen05.mma.ws.* variants do not need collector-b appended.
1375 if (Name.starts_with("ws"))
1376 return Intrinsic::not_intrinsic;
1377
1378 return F->getIntrinsicID();
1379}
1380
1381static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1382 return Name.consume_front("local") || Name.consume_front("shared") ||
1383 Name.consume_front("global") || Name.consume_front("constant") ||
1384 Name.consume_front("param");
1385}
1386
1387static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1388 if (!Name.consume_front("vp."))
1389 return 0;
1390 return StringSwitch<unsigned>(Name)
1391 .StartsWith("select", Instruction::Select)
1392 .StartsWith("add", Instruction::Add)
1393 .StartsWith("sub", Instruction::Sub)
1394 .StartsWith("mul", Instruction::Mul)
1395 .StartsWith("ashr", Instruction::AShr)
1396 .StartsWith("lshr", Instruction::LShr)
1397 .StartsWith("shl", Instruction::Shl)
1398 .StartsWith("or", Instruction::Or)
1399 .StartsWith("and", Instruction::And)
1400 .StartsWith("xor", Instruction::Xor)
1401 .StartsWith("fadd", Instruction::FAdd)
1402 .StartsWith("fsub", Instruction::FSub)
1403 .StartsWith("fmuladd", 0)
1404 .StartsWith("fmul", Instruction::FMul)
1405 .StartsWith("fdiv", Instruction::FDiv)
1406 .StartsWith("frem", Instruction::FRem)
1407 .StartsWith("fneg", Instruction::FNeg)
1408 .StartsWith("trunc", Instruction::Trunc)
1409 .StartsWith("zext", Instruction::ZExt)
1410 .StartsWith("sext", Instruction::SExt)
1411 .StartsWith("fptrunc", Instruction::FPTrunc)
1412 .StartsWith("fpext", Instruction::FPExt)
1413 .StartsWith("fptoui", Instruction::FPToUI)
1414 .StartsWith("fptosi", Instruction::FPToSI)
1415 .StartsWith("uitofp", Instruction::UIToFP)
1416 .StartsWith("sitofp", Instruction::SIToFP)
1417 .StartsWith("ptrtoint", Instruction::PtrToInt)
1418 .StartsWith("inttoptr", Instruction::IntToPtr)
1419 .StartsWith("icmp", Instruction::ICmp)
1420 .StartsWith("fcmp", Instruction::FCmp)
1421 .Default(0);
1422}
1423
1424static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1425 if (!Name.consume_front("vp."))
1426 return 0;
1427 return StringSwitch<Intrinsic::ID>(Name)
1428 .StartsWith("abs", Intrinsic::abs)
1429 .StartsWith("smax", Intrinsic::smax)
1430 .StartsWith("smin", Intrinsic::smin)
1431 .StartsWith("umax", Intrinsic::umax)
1432 .StartsWith("umin", Intrinsic::umin)
1433 .StartsWith("copysign", Intrinsic::copysign)
1434 .StartsWith("minnum", Intrinsic::minnum)
1435 .StartsWith("maxnum", Intrinsic::maxnum)
1436 .StartsWith("minimum", Intrinsic::minimum)
1437 .StartsWith("maximum", Intrinsic::maximum)
1438 .StartsWith("fabs", Intrinsic::fabs)
1439 .StartsWith("sqrt", Intrinsic::sqrt)
1440 .StartsWith("fma", Intrinsic::fma)
1441 .StartsWith("fmuladd", Intrinsic::fmuladd)
1442 .StartsWith("ceil", Intrinsic::ceil)
1443 .StartsWith("floor", Intrinsic::floor)
1444 .StartsWith("rint", Intrinsic::rint)
1445 .StartsWith("nearbyint", Intrinsic::nearbyint)
1446 .StartsWith("roundeven", Intrinsic::roundeven)
1447 .StartsWith("roundtozero", Intrinsic::trunc)
1448 .StartsWith("round", Intrinsic::round)
1449 .StartsWith("lrint", Intrinsic::lrint)
1450 .StartsWith("llrint", Intrinsic::llrint)
1451 .StartsWith("bitreverse", Intrinsic::bitreverse)
1452 .StartsWith("bswap", Intrinsic::bswap)
1453 .StartsWith("ctpop", Intrinsic::ctpop)
1454 .StartsWith("ctlz", Intrinsic::ctlz)
1455 .StartsWith("cttz.elts", 0)
1456 .StartsWith("cttz", Intrinsic::cttz)
1457 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1458 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1459 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1460 .StartsWith("usub.sat", Intrinsic::usub_sat)
1461 .StartsWith("fshl", Intrinsic::fshl)
1462 .StartsWith("fshr", Intrinsic::fshr)
1463 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1464 .Default(0);
1465}
1466
1470
1471static bool convertIntrinsicValidType(StringRef Name,
1472 const FunctionType *FuncTy) {
1473 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1474 if (Name.starts_with("to.fp16")) {
1475 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1476 HalfTy) &&
1477 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1478 FuncTy->getReturnType());
1479 }
1480
1481 if (Name.starts_with("from.fp16")) {
1482 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1483 HalfTy) &&
1484 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1485 FuncTy->getReturnType());
1486 }
1487
1488 return false;
1489}
1490
1491static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1492 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1493 if (IID == Intrinsic::not_intrinsic)
1494 return false;
1495
1496 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1497 if (Defaults.empty())
1498 return false;
1499
1500 // Overloaded intrinsics are out of scope for the default-arg feature
1501 // and will be supported in a follow-up.
1502 if (Intrinsic::isOverloaded(IID))
1503 return false;
1504
1505 // Get the canonical full declaration for this intrinsic.
1506 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1507
1508 // If the existing declaration already has all args, nothing to upgrade
1509 if (F->arg_size() >= FullDecl->arg_size())
1510 return false;
1511
1512 // Defaults are a contiguous trailing block, so checking the first missing
1513 // argument is enough.
1514 if (F->arg_size() < FirstDefault)
1515 return false;
1516
1517 NewFn = FullDecl;
1518 return true;
1519}
1520
1521static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1522 bool CanUpgradeDebugIntrinsicsToRecords) {
1523 assert(F && "Illegal to upgrade a non-existent Function.");
1524
1525 StringRef Name = F->getName();
1526
1527 // Quickly eliminate it, if it's not a candidate.
1528 if (!Name.consume_front("llvm.") || Name.empty())
1529 return false;
1530
1531 switch (Name[0]) {
1532 default: break;
1533 case 'a': {
1534 bool IsArm = Name.consume_front("arm.");
1535 if (IsArm || Name.consume_front("aarch64.")) {
1536 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1537 return true;
1538 break;
1539 }
1540
1541 if (Name.consume_front("amdgcn.")) {
1542 if (Name == "alignbit") {
1543 // Target specific intrinsic became redundant
1544 NewFn = Intrinsic::getOrInsertDeclaration(
1545 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1546 return true;
1547 }
1548
1549 if (Name.consume_front("atomic.")) {
1550 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1551 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1552 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1553 // and usub_sat so there's no new declaration.
1554 NewFn = nullptr;
1555 return true;
1556 }
1557 break; // No other 'amdgcn.atomic.*'
1558 }
1559
1560 switch (F->getIntrinsicID()) {
1561 default:
1562 break;
1563 // Legacy wmma iu intrinsics without the optional clamp operand.
1564 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1565 if (F->arg_size() == 7) {
1566 NewFn = nullptr;
1567 return true;
1568 }
1569 break;
1570 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1571 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1572 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1573 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1574 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1575 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1576 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1577 if (F->arg_size() == 8) {
1578 NewFn = nullptr;
1579 return true;
1580 }
1581 break;
1582 }
1583
1584 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1585 Name.consume_front("flat.atomic.")) {
1586 if (Name.starts_with("fadd") ||
1587 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1588 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1589 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1590 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1591 // declaration.
1592 NewFn = nullptr;
1593 return true;
1594 }
1595 }
1596
1597 if (Name.starts_with("ldexp.")) {
1598 // Target specific intrinsic became redundant
1599 NewFn = Intrinsic::getOrInsertDeclaration(
1600 F->getParent(), Intrinsic::ldexp,
1601 {F->getReturnType(), F->getArg(1)->getType()});
1602 return true;
1603 }
1604 break; // No other 'amdgcn.*'
1605 }
1606
1607 break;
1608 }
1609 case 'c': {
1610 if (F->arg_size() == 1) {
1611 if (Name.consume_front("convert.")) {
1612 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1613 NewFn = nullptr;
1614 return true;
1615 }
1616 }
1617
1618 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1619 .StartsWith("ctlz.", Intrinsic::ctlz)
1620 .StartsWith("cttz.", Intrinsic::cttz)
1621 .Default(Intrinsic::not_intrinsic);
1622 if (ID != Intrinsic::not_intrinsic) {
1623 rename(F);
1624 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1625 F->arg_begin()->getType());
1626 return true;
1627 }
1628 }
1629
1630 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1631 if (Name == "coro.end" &&
1632 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1633 CoroEndID = Intrinsic::coro_end;
1634 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1635 CoroEndID = Intrinsic::coro_end_async;
1636
1637 if (CoroEndID != Intrinsic::not_intrinsic) {
1638 rename(F);
1639 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1640 return true;
1641 }
1642
1643 break;
1644 }
1645 case 'd':
1646 if (Name.consume_front("dbg.")) {
1647 // Mark debug intrinsics for upgrade to new debug format.
1648 if (CanUpgradeDebugIntrinsicsToRecords) {
1649 if (Name == "addr" || Name == "value" || Name == "assign" ||
1650 Name == "declare" || Name == "label") {
1651 // There's no function to replace these with.
1652 NewFn = nullptr;
1653 // But we do want these to get upgraded.
1654 return true;
1655 }
1656 }
1657 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1658 // converted to DbgVariableRecords later.
1659 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1660 rename(F);
1661 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1662 Intrinsic::dbg_value);
1663 return true;
1664 }
1665 break; // No other 'dbg.*'.
1666 }
1667 break;
1668 case 'e':
1669 if (Name.consume_front("experimental.vector.")) {
1670 Intrinsic::ID ID =
1671 StringSwitch<Intrinsic::ID>(Name)
1672 // Skip over extract.last.active, otherwise it will be 'upgraded'
1673 // to a regular vector extract which is a different operation.
1674 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1675 .StartsWith("extract.", Intrinsic::vector_extract)
1676 .StartsWith("insert.", Intrinsic::vector_insert)
1677 .StartsWith("reverse.", Intrinsic::vector_reverse)
1678 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1679 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1680 .StartsWith("partial.reduce.add",
1681 Intrinsic::vector_partial_reduce_add)
1682 .Default(Intrinsic::not_intrinsic);
1683 if (ID != Intrinsic::not_intrinsic) {
1684 const auto *FT = F->getFunctionType();
1685 SmallVector<Type *, 2> Tys;
1686 if (ID == Intrinsic::vector_extract ||
1687 ID == Intrinsic::vector_interleave2)
1688 // Extracting overloads the return type.
1689 Tys.push_back(FT->getReturnType());
1690 if (ID != Intrinsic::vector_interleave2)
1691 Tys.push_back(FT->getParamType(0));
1692 if (ID == Intrinsic::vector_insert ||
1693 ID == Intrinsic::vector_partial_reduce_add)
1694 // Inserting overloads the inserted type.
1695 Tys.push_back(FT->getParamType(1));
1696 rename(F);
1697 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1698 return true;
1699 }
1700
1701 if (Name.consume_front("reduce.")) {
1702 SmallVector<StringRef, 2> Groups;
1703 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1704 if (R.match(Name, &Groups))
1705 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1706 .Case("add", Intrinsic::vector_reduce_add)
1707 .Case("mul", Intrinsic::vector_reduce_mul)
1708 .Case("and", Intrinsic::vector_reduce_and)
1709 .Case("or", Intrinsic::vector_reduce_or)
1710 .Case("xor", Intrinsic::vector_reduce_xor)
1711 .Case("smax", Intrinsic::vector_reduce_smax)
1712 .Case("smin", Intrinsic::vector_reduce_smin)
1713 .Case("umax", Intrinsic::vector_reduce_umax)
1714 .Case("umin", Intrinsic::vector_reduce_umin)
1715 .Case("fmax", Intrinsic::vector_reduce_fmax)
1716 .Case("fmin", Intrinsic::vector_reduce_fmin)
1717 .Default(Intrinsic::not_intrinsic);
1718
1719 bool V2 = false;
1720 if (ID == Intrinsic::not_intrinsic) {
1721 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1722 Groups.clear();
1723 V2 = true;
1724 if (R2.match(Name, &Groups))
1725 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1726 .Case("fadd", Intrinsic::vector_reduce_fadd)
1727 .Case("fmul", Intrinsic::vector_reduce_fmul)
1728 .Default(Intrinsic::not_intrinsic);
1729 }
1730 if (ID != Intrinsic::not_intrinsic) {
1731 rename(F);
1732 auto Args = F->getFunctionType()->params();
1733 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1734 {Args[V2 ? 1 : 0]});
1735 return true;
1736 }
1737 break; // No other 'expermental.vector.reduce.*'.
1738 }
1739
1740 if (Name.consume_front("splice"))
1741 return true;
1742 break; // No other 'experimental.vector.*'.
1743 }
1744 if (Name.consume_front("experimental.stepvector.")) {
1745 Intrinsic::ID ID = Intrinsic::stepvector;
1746 rename(F);
1747 NewFn = Intrinsic::getOrInsertDeclaration(
1748 F->getParent(), ID, F->getFunctionType()->getReturnType());
1749 return true;
1750 }
1751 break; // No other 'e*'.
1752 case 'f':
1753 if (Name.starts_with("flt.rounds")) {
1754 rename(F);
1755 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1756 Intrinsic::get_rounding);
1757 return true;
1758 }
1759 break;
1760 case 'i':
1761 if (Name.starts_with("invariant.group.barrier")) {
1762 // Rename invariant.group.barrier to launder.invariant.group
1763 auto Args = F->getFunctionType()->params();
1764 Type* ObjectPtr[1] = {Args[0]};
1765 rename(F);
1766 NewFn = Intrinsic::getOrInsertDeclaration(
1767 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1768 return true;
1769 }
1770 break;
1771 case 'l': {
1772 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1773 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1774 if (IsLifetimeStart || IsLifetimeEnd) {
1775 if (F->arg_size() == 2) {
1776 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1777 : Intrinsic::lifetime_end;
1778 rename(F);
1779 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1780 // arguments. Use the Ptr argument to create new declaration.
1781 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1782 F->getArg(1)->getType());
1783 return true;
1784 } else if (F->arg_size() == 1 && Name == ".i64") {
1785 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1786 // Autoupgrade prior to
1787 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1788 // intrinsic with no expected calls. To allow auto-upgrade process to
1789 // delete such invalid intrinsic declaration, set NewFn = nullptr
1790 // and return true here. If there are actual calls to this intrinsic
1791 // (which is not expected), they will be deleted in
1792 // UpgradeIntrinsicCall.
1793 NewFn = nullptr;
1794 return true;
1795 }
1796 }
1797 break;
1798 }
1799 case 'm': {
1800 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1801 // alignment parameter to embedding the alignment as an attribute of
1802 // the pointer args.
1803 if (unsigned ID = StringSwitch<unsigned>(Name)
1804 .StartsWith("memcpy.", Intrinsic::memcpy)
1805 .StartsWith("memmove.", Intrinsic::memmove)
1806 .Default(0)) {
1807 if (F->arg_size() == 5) {
1808 rename(F);
1809 // Get the types of dest, src, and len
1810 ArrayRef<Type *> ParamTypes =
1811 F->getFunctionType()->params().slice(0, 3);
1812 NewFn =
1813 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1814 return true;
1815 }
1816 }
1817 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1818 rename(F);
1819 // Get the types of dest, and len
1820 const auto *FT = F->getFunctionType();
1821 Type *ParamTypes[2] = {
1822 FT->getParamType(0), // Dest
1823 FT->getParamType(2) // len
1824 };
1825 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1826 Intrinsic::memset, ParamTypes);
1827 return true;
1828 }
1829
1830 unsigned MaskedID =
1831 StringSwitch<unsigned>(Name)
1832 .StartsWith("masked.load", Intrinsic::masked_load)
1833 .StartsWith("masked.gather", Intrinsic::masked_gather)
1834 .StartsWith("masked.store", Intrinsic::masked_store)
1835 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1836 .Default(0);
1837 if (MaskedID && F->arg_size() == 4) {
1838 rename(F);
1839 if (MaskedID == Intrinsic::masked_load ||
1840 MaskedID == Intrinsic::masked_gather) {
1841 NewFn = Intrinsic::getOrInsertDeclaration(
1842 F->getParent(), MaskedID,
1843 {F->getReturnType(), F->getArg(0)->getType()});
1844 return true;
1845 }
1846 NewFn = Intrinsic::getOrInsertDeclaration(
1847 F->getParent(), MaskedID,
1848 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1849 return true;
1850 }
1851 break;
1852 }
1853 case 'n': {
1854 if (Name.consume_front("nvvm.")) {
1855 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1856 if (F->arg_size() == 1) {
1857 Intrinsic::ID IID =
1858 StringSwitch<Intrinsic::ID>(Name)
1859 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1860 .Case("clz.i", Intrinsic::ctlz)
1861 .Case("popc.i", Intrinsic::ctpop)
1862 .Default(Intrinsic::not_intrinsic);
1863 if (IID != Intrinsic::not_intrinsic) {
1864 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1865 {F->getReturnType()});
1866 return true;
1867 }
1868 } else if (F->arg_size() == 2) {
1869 Intrinsic::ID IID =
1870 StringSwitch<Intrinsic::ID>(Name)
1871 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1872 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1873 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1874 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1875 .Default(Intrinsic::not_intrinsic);
1876 if (IID != Intrinsic::not_intrinsic) {
1877 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1878 {F->getReturnType()});
1879 return true;
1880 }
1881 }
1882
1883 // Check for nvvm intrinsics that need a return type adjustment.
1884 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1885 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1886 if (IID != Intrinsic::not_intrinsic) {
1887 NewFn = nullptr;
1888 return true;
1889 }
1890 }
1891
1892 // Upgrade Distributed Shared Memory Intrinsics
1893 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1894 if (IID != Intrinsic::not_intrinsic) {
1895 rename(F);
1896 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1897 return true;
1898 }
1899
1900 // Upgrade TMA reduction intrinsics
1901 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
1902 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
1903 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
1904 if (IID != Intrinsic::not_intrinsic) {
1905 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1906 return true;
1907 }
1908
1909 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
1910 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
1911 if (IID != Intrinsic::not_intrinsic) {
1912 rename(F);
1913 NewFn = Intrinsic::getOrInsertDeclaration(
1914 F->getParent(), IID, F->getReturnType(),
1915 F->getFunctionType()->params());
1916 return true;
1917 }
1918
1919 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
1920 // tcgen05.alloc shared variants to anyptr intrinsics.
1921 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
1922 if (IID != Intrinsic::not_intrinsic) {
1923 rename(F);
1924 if (Intrinsic::isOverloaded(IID))
1925 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1926 {F->getArg(0)->getType()});
1927 else
1928 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1929 return true;
1930 }
1931
1932 // Upgrade TMA copy G2S Intrinsics
1933 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1934 if (IID != Intrinsic::not_intrinsic) {
1935 rename(F);
1936 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1937 return true;
1938 }
1939
1940 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
1941 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
1942 if (IID != Intrinsic::not_intrinsic) {
1943 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1944 return NewFn != F;
1945 }
1946
1947 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1948 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1949 //
1950 // TODO: We could add lohi.i2d.
1951 bool Expand = false;
1952 if (Name.consume_front("abs."))
1953 // nvvm.abs.{i,ii}
1954 Expand =
1955 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1956 else if (Name.consume_front("fabs."))
1957 // nvvm.fabs.{f,ftz.f,d}
1958 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1959 else if (Name.consume_front("ex2.approx."))
1960 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1961 Expand =
1962 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1963 else if (Name.consume_front("atomic.load."))
1964 // nvvm.atomic.load.add.{f32,f64}.p
1965 // nvvm.atomic.load.{inc,dec}.32.p
1966 Expand = StringSwitch<bool>(Name)
1967 .StartsWith("add.f32.p", true)
1968 .StartsWith("add.f64.p", true)
1969 .StartsWith("inc.32.p", true)
1970 .StartsWith("dec.32.p", true)
1971 .Default(false);
1972 else if (Name.consume_front("atomic."))
1973 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
1974 // nvvm.atomic.cas.gen.i.{cta,sys}
1975 Expand = StringSwitch<bool>(Name)
1976 .StartsWith("add.gen.", true)
1977 .StartsWith("exch.gen.", true)
1978 .StartsWith("max.gen.", true)
1979 .StartsWith("min.gen.", true)
1980 .StartsWith("inc.gen.", true)
1981 .StartsWith("dec.gen.", true)
1982 .StartsWith("and.gen.", true)
1983 .StartsWith("or.gen.", true)
1984 .StartsWith("xor.gen.", true)
1985 .StartsWith("cas.gen.", true)
1986 .Default(false);
1987 else if (Name.consume_front("bitcast."))
1988 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
1989 Expand =
1990 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
1991 else if (Name.consume_front("rotate."))
1992 // nvvm.rotate.{b32,b64,right.b64}
1993 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
1994 else if (Name.consume_front("ptr.gen.to."))
1995 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
1996 Expand = consumeNVVMPtrAddrSpace(Name);
1997 else if (Name.consume_front("ptr."))
1998 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
1999 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2000 else if (Name.consume_front("ldg.global."))
2001 // nvvm.ldg.global.{i,p,f}
2002 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2003 Name.starts_with("p."));
2004 else
2005 Expand = StringSwitch<bool>(Name)
2006 .Case("barrier0", true)
2007 .Case("barrier.n", true)
2008 .Case("barrier.sync.cnt", true)
2009 .Case("barrier.sync", true)
2010 .Case("barrier", true)
2011 .Case("bar.sync", true)
2012 .Case("barrier0.popc", true)
2013 .Case("barrier0.and", true)
2014 .Case("barrier0.or", true)
2015 .Case("clz.ll", true)
2016 .Case("popc.ll", true)
2017 .Case("h2f", true)
2018 .Case("swap.lo.hi.b64", true)
2019 .Case("tanh.approx.f32", true)
2020 .Default(false);
2021
2022 if (Expand) {
2023 NewFn = nullptr;
2024 return true;
2025 }
2026 break; // No other 'nvvm.*'.
2027 }
2028 break;
2029 }
2030 case 'o':
2031 if (Name.starts_with("objectsize.")) {
2032 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2033 if (F->arg_size() == 2 || F->arg_size() == 3) {
2034 rename(F);
2035 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2036 Intrinsic::objectsize, Tys);
2037 return true;
2038 }
2039 }
2040 break;
2041
2042 case 'p':
2043 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2044 rename(F);
2045 NewFn = Intrinsic::getOrInsertDeclaration(
2046 F->getParent(), Intrinsic::ptr_annotation,
2047 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2048 return true;
2049 }
2050 break;
2051
2052 case 'r': {
2053 if (Name.consume_front("riscv.")) {
2054 Intrinsic::ID ID;
2055 ID = StringSwitch<Intrinsic::ID>(Name)
2056 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2057 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2058 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2059 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2060 .Default(Intrinsic::not_intrinsic);
2061 if (ID != Intrinsic::not_intrinsic) {
2062 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2063 rename(F);
2064 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2065 return true;
2066 }
2067 break; // No other applicable upgrades.
2068 }
2069
2070 ID = StringSwitch<Intrinsic::ID>(Name)
2071 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2072 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2073 .Default(Intrinsic::not_intrinsic);
2074 if (ID != Intrinsic::not_intrinsic) {
2075 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2076 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2077 rename(F);
2078 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2079 return true;
2080 }
2081 break; // No other applicable upgrades.
2082 }
2083
2084 ID = StringSwitch<Intrinsic::ID>(Name)
2085 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2086 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2087 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2088 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2089 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2090 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2091 .Default(Intrinsic::not_intrinsic);
2092 if (ID != Intrinsic::not_intrinsic) {
2093 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2094 rename(F);
2095 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2096 return true;
2097 }
2098 break; // No other applicable upgrades.
2099 }
2100
2101 // Replace llvm.riscv.clmul with llvm.clmul.
2102 if (Name == "clmul.i32" || Name == "clmul.i64") {
2103 NewFn = Intrinsic::getOrInsertDeclaration(
2104 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2105 return true;
2106 }
2107
2108 break; // No other 'riscv.*' intrinsics
2109 }
2110 } break;
2111
2112 case 's':
2113 if (Name == "stackprotectorcheck") {
2114 NewFn = nullptr;
2115 return true;
2116 }
2117 break;
2118
2119 case 't':
2120 if (Name == "thread.pointer") {
2121 NewFn = Intrinsic::getOrInsertDeclaration(
2122 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2123 return true;
2124 }
2125 break;
2126
2127 case 'v': {
2128 if (Name == "var.annotation" && F->arg_size() == 4) {
2129 rename(F);
2130 NewFn = Intrinsic::getOrInsertDeclaration(
2131 F->getParent(), Intrinsic::var_annotation,
2132 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2133 return true;
2134 }
2135 if (Name.consume_front("vector.splice")) {
2136 if (Name.starts_with(".left") || Name.starts_with(".right"))
2137 break;
2138 return true;
2139 }
2140 if (shouldUpgradeVPIntrinsic(Name))
2141 return true;
2142 break;
2143 }
2144
2145 case 'w':
2146 if (Name.consume_front("wasm.")) {
2147 Intrinsic::ID ID =
2148 StringSwitch<Intrinsic::ID>(Name)
2149 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2150 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2151 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2152 .Default(Intrinsic::not_intrinsic);
2153 if (ID != Intrinsic::not_intrinsic) {
2154 rename(F);
2155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2156 F->getReturnType());
2157 return true;
2158 }
2159
2160 if (Name.consume_front("dot.i8x16.i7x16.")) {
2161 ID = StringSwitch<Intrinsic::ID>(Name)
2162 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2163 .Case("add.signed",
2164 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2165 .Default(Intrinsic::not_intrinsic);
2166 if (ID != Intrinsic::not_intrinsic) {
2167 rename(F);
2168 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2169 return true;
2170 }
2171 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2172 }
2173 break; // No other 'wasm.*'.
2174 }
2175 break;
2176
2177 case 'x':
2178 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2179 return true;
2180 }
2181
2182 auto *ST = dyn_cast<StructType>(F->getReturnType());
2183 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2184 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2185 // Replace return type with literal non-packed struct. Only do this for
2186 // intrinsics declared to return a struct, not for intrinsics with
2187 // overloaded return type, in which case the exact struct type will be
2188 // mangled into the name.
2189 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2190 FunctionType *FT = F->getFunctionType();
2191 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2192 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2193 std::string Name = F->getName().str();
2194 rename(F);
2195 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2196 Name, F->getParent());
2197
2198 // The new function may also need remangling.
2199 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2200 NewFn = *Result;
2201 return true;
2202 }
2203 }
2204
2205 // Remangle our intrinsic since we upgrade the mangling
2206 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2207 if (Result != std::nullopt) {
2208 NewFn = *Result;
2209 return true;
2210 }
2211
2212 // This may not belong here. This function is effectively being overloaded
2213 // to both detect an intrinsic which needs upgrading, and to provide the
2214 // upgraded form of the intrinsic. We should perhaps have two separate
2215 // functions for this.
2216 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
2217 return true;
2218
2219 return false;
2220}
2221
2222bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2223 bool CanUpgradeDebugIntrinsicsToRecords) {
2224 NewFn = nullptr;
2225 bool Upgraded =
2226 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2227
2228 // Upgrade intrinsic attributes. This does not change the function.
2229 if (NewFn)
2230 F = NewFn;
2231 if (Intrinsic::ID id = F->getIntrinsicID()) {
2232 // Only do this if the intrinsic signature is valid.
2233 SmallVector<Type *> OverloadTys;
2234 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2235 F->setAttributes(
2236 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2237 }
2238 return Upgraded;
2239}
2240
2241GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2242 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2243 GV->getName() == "llvm.global_dtors")) ||
2244 !GV->hasInitializer())
2245 return nullptr;
2246 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2247 if (!ATy)
2248 return nullptr;
2249 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2250 if (!STy || STy->getNumElements() != 2)
2251 return nullptr;
2252
2253 LLVMContext &C = GV->getContext();
2254 IRBuilder<> IRB(C);
2255 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2256 IRB.getPtrTy());
2257 Constant *Init = GV->getInitializer();
2258 unsigned N = Init->getNumOperands();
2259 std::vector<Constant *> NewCtors(N);
2260 for (unsigned i = 0; i != N; ++i) {
2261 auto Ctor = cast<Constant>(Init->getOperand(i));
2262 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2263 Ctor->getAggregateElement(1),
2264 ConstantPointerNull::get(IRB.getPtrTy()));
2265 }
2266 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2267
2268 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2269 NewInit, GV->getName());
2270}
2271
2272// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2273// to byte shuffles.
2274static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2275 unsigned Shift) {
2276 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2277 unsigned NumElts = ResultTy->getNumElements() * 8;
2278
2279 // Bitcast from a 64-bit element type to a byte element type.
2280 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2281 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2282
2283 // We'll be shuffling in zeroes.
2284 Value *Res = Constant::getNullValue(VecTy);
2285
2286 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2287 // we'll just return the zero vector.
2288 if (Shift < 16) {
2289 int Idxs[64];
2290 // 256/512-bit version is split into 2/4 16-byte lanes.
2291 for (unsigned l = 0; l != NumElts; l += 16)
2292 for (unsigned i = 0; i != 16; ++i) {
2293 unsigned Idx = NumElts + i - Shift;
2294 if (Idx < NumElts)
2295 Idx -= NumElts - 16; // end of lane, switch operand.
2296 Idxs[l + i] = Idx + l;
2297 }
2298
2299 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2300 }
2301
2302 // Bitcast back to a 64-bit element type.
2303 return Builder.CreateBitCast(Res, ResultTy, "cast");
2304}
2305
2306// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2307// to byte shuffles.
2308static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2309 unsigned Shift) {
2310 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2311 unsigned NumElts = ResultTy->getNumElements() * 8;
2312
2313 // Bitcast from a 64-bit element type to a byte element type.
2314 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2315 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2316
2317 // We'll be shuffling in zeroes.
2318 Value *Res = Constant::getNullValue(VecTy);
2319
2320 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2321 // we'll just return the zero vector.
2322 if (Shift < 16) {
2323 int Idxs[64];
2324 // 256/512-bit version is split into 2/4 16-byte lanes.
2325 for (unsigned l = 0; l != NumElts; l += 16)
2326 for (unsigned i = 0; i != 16; ++i) {
2327 unsigned Idx = i + Shift;
2328 if (Idx >= 16)
2329 Idx += NumElts - 16; // end of lane, switch operand.
2330 Idxs[l + i] = Idx + l;
2331 }
2332
2333 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2334 }
2335
2336 // Bitcast back to a 64-bit element type.
2337 return Builder.CreateBitCast(Res, ResultTy, "cast");
2338}
2339
2340static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2341 unsigned NumElts) {
2342 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2343 llvm::VectorType *MaskTy = FixedVectorType::get(
2344 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2345 Mask = Builder.CreateBitCast(Mask, MaskTy);
2346
2347 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2348 // i8 and we need to extract down to the right number of elements.
2349 if (NumElts <= 4) {
2350 int Indices[4];
2351 for (unsigned i = 0; i != NumElts; ++i)
2352 Indices[i] = i;
2353 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2354 "extract");
2355 }
2356
2357 return Mask;
2358}
2359
2360static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2361 Value *Op1) {
2362 // If the mask is all ones just emit the first operation.
2363 if (const auto *C = dyn_cast<Constant>(Mask))
2364 if (C->isAllOnesValue())
2365 return Op0;
2366
2367 Mask = getX86MaskVec(Builder, Mask,
2368 cast<FixedVectorType>(Op0->getType())->getNumElements());
2369 return Builder.CreateSelect(Mask, Op0, Op1);
2370}
2371
2372static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2373 Value *Op1) {
2374 // If the mask is all ones just emit the first operation.
2375 if (const auto *C = dyn_cast<Constant>(Mask))
2376 if (C->isAllOnesValue())
2377 return Op0;
2378
2379 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2380 Mask->getType()->getIntegerBitWidth());
2381 Mask = Builder.CreateBitCast(Mask, MaskTy);
2382 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2383 return Builder.CreateSelect(Mask, Op0, Op1);
2384}
2385
2386// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2387// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2388// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2389static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2390 Value *Op1, Value *Shift,
2391 Value *Passthru, Value *Mask,
2392 bool IsVALIGN) {
2393 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2394
2395 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2396 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2397 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2398 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2399
2400 // Mask the immediate for VALIGN.
2401 if (IsVALIGN)
2402 ShiftVal &= (NumElts - 1);
2403
2404 // If palignr is shifting the pair of vectors more than the size of two
2405 // lanes, emit zero.
2406 if (ShiftVal >= 32)
2407 return llvm::Constant::getNullValue(Op0->getType());
2408
2409 // If palignr is shifting the pair of input vectors more than one lane,
2410 // but less than two lanes, convert to shifting in zeroes.
2411 if (ShiftVal > 16) {
2412 ShiftVal -= 16;
2413 Op1 = Op0;
2414 Op0 = llvm::Constant::getNullValue(Op0->getType());
2415 }
2416
2417 int Indices[64];
2418 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2419 for (unsigned l = 0; l < NumElts; l += 16) {
2420 for (unsigned i = 0; i != 16; ++i) {
2421 unsigned Idx = ShiftVal + i;
2422 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2423 Idx += NumElts - 16; // End of lane, switch operand.
2424 Indices[l + i] = Idx + l;
2425 }
2426 }
2427
2428 Value *Align = Builder.CreateShuffleVector(
2429 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2430
2431 return emitX86Select(Builder, Mask, Align, Passthru);
2432}
2433
2434static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2435 bool ZeroMask, bool IndexForm) {
2436 Type *Ty = CI.getType();
2437 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2438 unsigned EltWidth = Ty->getScalarSizeInBits();
2439 bool IsFloat = Ty->isFPOrFPVectorTy();
2440 Intrinsic::ID IID;
2441 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2442 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2443 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2444 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2445 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2446 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2447 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2448 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2449 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2450 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2451 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2452 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2453 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2454 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2455 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2456 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2457 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2458 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2459 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2460 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2461 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2462 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2463 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2464 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2465 else if (VecWidth == 128 && EltWidth == 16)
2466 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2467 else if (VecWidth == 256 && EltWidth == 16)
2468 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2469 else if (VecWidth == 512 && EltWidth == 16)
2470 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2471 else if (VecWidth == 128 && EltWidth == 8)
2472 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2473 else if (VecWidth == 256 && EltWidth == 8)
2474 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2475 else if (VecWidth == 512 && EltWidth == 8)
2476 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2477 else
2478 llvm_unreachable("Unexpected intrinsic");
2479
2480 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2481 CI.getArgOperand(2) };
2482
2483 // If this isn't index form we need to swap operand 0 and 1.
2484 if (!IndexForm)
2485 std::swap(Args[0], Args[1]);
2486
2487 Value *V = Builder.CreateIntrinsic(IID, Args);
2488 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2489 : Builder.CreateBitCast(CI.getArgOperand(1),
2490 Ty);
2491 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2492}
2493
2494static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2495 Intrinsic::ID IID) {
2496 Type *Ty = CI.getType();
2497 Value *Op0 = CI.getOperand(0);
2498 Value *Op1 = CI.getOperand(1);
2499 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2500
2501 if (CI.arg_size() == 4) { // For masked intrinsics.
2502 Value *VecSrc = CI.getOperand(2);
2503 Value *Mask = CI.getOperand(3);
2504 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2505 }
2506 return Res;
2507}
2508
2509static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2510 bool IsRotateRight) {
2511 Type *Ty = CI.getType();
2512 Value *Src = CI.getArgOperand(0);
2513 Value *Amt = CI.getArgOperand(1);
2514
2515 // Amount may be scalar immediate, in which case create a splat vector.
2516 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2517 // we only care about the lowest log2 bits anyway.
2518 if (Amt->getType() != Ty) {
2519 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2520 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2521 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2522 }
2523
2524 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2525 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2526
2527 if (CI.arg_size() == 4) { // For masked intrinsics.
2528 Value *VecSrc = CI.getOperand(2);
2529 Value *Mask = CI.getOperand(3);
2530 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2531 }
2532 return Res;
2533}
2534
2535static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2536 bool IsSigned) {
2537 Type *Ty = CI.getType();
2538 Value *LHS = CI.getArgOperand(0);
2539 Value *RHS = CI.getArgOperand(1);
2540
2541 CmpInst::Predicate Pred;
2542 switch (Imm) {
2543 case 0x0:
2544 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2545 break;
2546 case 0x1:
2547 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2548 break;
2549 case 0x2:
2550 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2551 break;
2552 case 0x3:
2553 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2554 break;
2555 case 0x4:
2556 Pred = ICmpInst::ICMP_EQ;
2557 break;
2558 case 0x5:
2559 Pred = ICmpInst::ICMP_NE;
2560 break;
2561 case 0x6:
2562 return Constant::getNullValue(Ty); // FALSE
2563 case 0x7:
2564 return Constant::getAllOnesValue(Ty); // TRUE
2565 default:
2566 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2567 }
2568
2569 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2570 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2571 return Ext;
2572}
2573
2574static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2575 bool IsShiftRight, bool ZeroMask) {
2576 Type *Ty = CI.getType();
2577 Value *Op0 = CI.getArgOperand(0);
2578 Value *Op1 = CI.getArgOperand(1);
2579 Value *Amt = CI.getArgOperand(2);
2580
2581 if (IsShiftRight)
2582 std::swap(Op0, Op1);
2583
2584 // Amount may be scalar immediate, in which case create a splat vector.
2585 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2586 // we only care about the lowest log2 bits anyway.
2587 if (Amt->getType() != Ty) {
2588 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2589 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2590 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2591 }
2592
2593 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2594 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2595
2596 unsigned NumArgs = CI.arg_size();
2597 if (NumArgs >= 4) { // For masked intrinsics.
2598 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2599 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2600 CI.getArgOperand(0);
2601 Value *Mask = CI.getOperand(NumArgs - 1);
2602 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2603 }
2604 return Res;
2605}
2606
2607static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2608 Value *Mask, bool Aligned) {
2609 const Align Alignment =
2610 Aligned
2611 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2612 : Align(1);
2613
2614 // If the mask is all ones just emit a regular store.
2615 if (const auto *C = dyn_cast<Constant>(Mask))
2616 if (C->isAllOnesValue())
2617 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2618
2619 // Convert the mask from an integer type to a vector of i1.
2620 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2621 Mask = getX86MaskVec(Builder, Mask, NumElts);
2622 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2623}
2624
2625static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2626 Value *Passthru, Value *Mask, bool Aligned) {
2627 Type *ValTy = Passthru->getType();
2628 const Align Alignment =
2629 Aligned
2630 ? Align(
2631 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2632 8)
2633 : Align(1);
2634
2635 // If the mask is all ones just emit a regular store.
2636 if (const auto *C = dyn_cast<Constant>(Mask))
2637 if (C->isAllOnesValue())
2638 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2639
2640 // Convert the mask from an integer type to a vector of i1.
2641 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2642 Mask = getX86MaskVec(Builder, Mask, NumElts);
2643 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2644}
2645
2646static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2647 Type *Ty = CI.getType();
2648 Value *Op0 = CI.getArgOperand(0);
2649 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2650 {Op0, Builder.getInt1(false)});
2651 if (CI.arg_size() == 3)
2652 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2653 return Res;
2654}
2655
2656static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2657 Type *Ty = CI.getType();
2658
2659 // Arguments have a vXi32 type so cast to vXi64.
2660 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2661 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2662
2663 if (IsSigned) {
2664 // Shift left then arithmetic shift right.
2665 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2666 LHS = Builder.CreateShl(LHS, ShiftAmt);
2667 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2668 RHS = Builder.CreateShl(RHS, ShiftAmt);
2669 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2670 } else {
2671 // Clear the upper bits.
2672 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2673 LHS = Builder.CreateAnd(LHS, Mask);
2674 RHS = Builder.CreateAnd(RHS, Mask);
2675 }
2676
2677 Value *Res = Builder.CreateMul(LHS, RHS);
2678
2679 if (CI.arg_size() == 4)
2680 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2681
2682 return Res;
2683}
2684
2685// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2686static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2687 Value *Mask) {
2688 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2689 if (Mask) {
2690 const auto *C = dyn_cast<Constant>(Mask);
2691 if (!C || !C->isAllOnesValue())
2692 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2693 }
2694
2695 if (NumElts < 8) {
2696 int Indices[8];
2697 for (unsigned i = 0; i != NumElts; ++i)
2698 Indices[i] = i;
2699 for (unsigned i = NumElts; i != 8; ++i)
2700 Indices[i] = NumElts + i % NumElts;
2701 Vec = Builder.CreateShuffleVector(Vec,
2702 Constant::getNullValue(Vec->getType()),
2703 Indices);
2704 }
2705 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2706}
2707
2708static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2709 unsigned CC, bool Signed) {
2710 Value *Op0 = CI.getArgOperand(0);
2711 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2712
2713 Value *Cmp;
2714 if (CC == 3) {
2715 Cmp = Constant::getNullValue(
2716 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2717 } else if (CC == 7) {
2718 Cmp = Constant::getAllOnesValue(
2719 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2720 } else {
2721 ICmpInst::Predicate Pred;
2722 switch (CC) {
2723 default: llvm_unreachable("Unknown condition code");
2724 case 0: Pred = ICmpInst::ICMP_EQ; break;
2725 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2726 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2727 case 4: Pred = ICmpInst::ICMP_NE; break;
2728 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2729 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2730 }
2731 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2732 }
2733
2734 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2735
2736 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2737}
2738
2739// Replace a masked intrinsic with an older unmasked intrinsic.
2740static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2741 Intrinsic::ID IID) {
2742 Value *Rep =
2743 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2744 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2745}
2746
2747static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2748 Value* A = CI.getArgOperand(0);
2749 Value* B = CI.getArgOperand(1);
2750 Value* Src = CI.getArgOperand(2);
2751 Value* Mask = CI.getArgOperand(3);
2752
2753 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2754 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2755 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2756 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2757 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2758 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2759}
2760
2761static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2762 Value* Op = CI.getArgOperand(0);
2763 Type* ReturnOp = CI.getType();
2764 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2765 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2766 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2767}
2768
2769// Replace intrinsic with unmasked version and a select.
2770static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2771 CallBase &CI, Value *&Rep) {
2772 Name = Name.substr(12); // Remove avx512.mask.
2773
2774 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2775 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2776 Intrinsic::ID IID;
2777 if (Name.starts_with("max.p")) {
2778 if (VecWidth == 128 && EltWidth == 32)
2779 IID = Intrinsic::x86_sse_max_ps;
2780 else if (VecWidth == 128 && EltWidth == 64)
2781 IID = Intrinsic::x86_sse2_max_pd;
2782 else if (VecWidth == 256 && EltWidth == 32)
2783 IID = Intrinsic::x86_avx_max_ps_256;
2784 else if (VecWidth == 256 && EltWidth == 64)
2785 IID = Intrinsic::x86_avx_max_pd_256;
2786 else
2787 llvm_unreachable("Unexpected intrinsic");
2788 } else if (Name.starts_with("min.p")) {
2789 if (VecWidth == 128 && EltWidth == 32)
2790 IID = Intrinsic::x86_sse_min_ps;
2791 else if (VecWidth == 128 && EltWidth == 64)
2792 IID = Intrinsic::x86_sse2_min_pd;
2793 else if (VecWidth == 256 && EltWidth == 32)
2794 IID = Intrinsic::x86_avx_min_ps_256;
2795 else if (VecWidth == 256 && EltWidth == 64)
2796 IID = Intrinsic::x86_avx_min_pd_256;
2797 else
2798 llvm_unreachable("Unexpected intrinsic");
2799 } else if (Name.starts_with("pshuf.b.")) {
2800 if (VecWidth == 128)
2801 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2802 else if (VecWidth == 256)
2803 IID = Intrinsic::x86_avx2_pshuf_b;
2804 else if (VecWidth == 512)
2805 IID = Intrinsic::x86_avx512_pshuf_b_512;
2806 else
2807 llvm_unreachable("Unexpected intrinsic");
2808 } else if (Name.starts_with("pmul.hr.sw.")) {
2809 if (VecWidth == 128)
2810 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2811 else if (VecWidth == 256)
2812 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2813 else if (VecWidth == 512)
2814 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2815 else
2816 llvm_unreachable("Unexpected intrinsic");
2817 } else if (Name.starts_with("pmulh.w.")) {
2818 if (VecWidth == 128)
2819 IID = Intrinsic::x86_sse2_pmulh_w;
2820 else if (VecWidth == 256)
2821 IID = Intrinsic::x86_avx2_pmulh_w;
2822 else if (VecWidth == 512)
2823 IID = Intrinsic::x86_avx512_pmulh_w_512;
2824 else
2825 llvm_unreachable("Unexpected intrinsic");
2826 } else if (Name.starts_with("pmulhu.w.")) {
2827 if (VecWidth == 128)
2828 IID = Intrinsic::x86_sse2_pmulhu_w;
2829 else if (VecWidth == 256)
2830 IID = Intrinsic::x86_avx2_pmulhu_w;
2831 else if (VecWidth == 512)
2832 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2833 else
2834 llvm_unreachable("Unexpected intrinsic");
2835 } else if (Name.starts_with("pmaddw.d.")) {
2836 if (VecWidth == 128)
2837 IID = Intrinsic::x86_sse2_pmadd_wd;
2838 else if (VecWidth == 256)
2839 IID = Intrinsic::x86_avx2_pmadd_wd;
2840 else if (VecWidth == 512)
2841 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2842 else
2843 llvm_unreachable("Unexpected intrinsic");
2844 } else if (Name.starts_with("pmaddubs.w.")) {
2845 if (VecWidth == 128)
2846 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2847 else if (VecWidth == 256)
2848 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2849 else if (VecWidth == 512)
2850 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2851 else
2852 llvm_unreachable("Unexpected intrinsic");
2853 } else if (Name.starts_with("packsswb.")) {
2854 if (VecWidth == 128)
2855 IID = Intrinsic::x86_sse2_packsswb_128;
2856 else if (VecWidth == 256)
2857 IID = Intrinsic::x86_avx2_packsswb;
2858 else if (VecWidth == 512)
2859 IID = Intrinsic::x86_avx512_packsswb_512;
2860 else
2861 llvm_unreachable("Unexpected intrinsic");
2862 } else if (Name.starts_with("packssdw.")) {
2863 if (VecWidth == 128)
2864 IID = Intrinsic::x86_sse2_packssdw_128;
2865 else if (VecWidth == 256)
2866 IID = Intrinsic::x86_avx2_packssdw;
2867 else if (VecWidth == 512)
2868 IID = Intrinsic::x86_avx512_packssdw_512;
2869 else
2870 llvm_unreachable("Unexpected intrinsic");
2871 } else if (Name.starts_with("packuswb.")) {
2872 if (VecWidth == 128)
2873 IID = Intrinsic::x86_sse2_packuswb_128;
2874 else if (VecWidth == 256)
2875 IID = Intrinsic::x86_avx2_packuswb;
2876 else if (VecWidth == 512)
2877 IID = Intrinsic::x86_avx512_packuswb_512;
2878 else
2879 llvm_unreachable("Unexpected intrinsic");
2880 } else if (Name.starts_with("packusdw.")) {
2881 if (VecWidth == 128)
2882 IID = Intrinsic::x86_sse41_packusdw;
2883 else if (VecWidth == 256)
2884 IID = Intrinsic::x86_avx2_packusdw;
2885 else if (VecWidth == 512)
2886 IID = Intrinsic::x86_avx512_packusdw_512;
2887 else
2888 llvm_unreachable("Unexpected intrinsic");
2889 } else if (Name.starts_with("vpermilvar.")) {
2890 if (VecWidth == 128 && EltWidth == 32)
2891 IID = Intrinsic::x86_avx_vpermilvar_ps;
2892 else if (VecWidth == 128 && EltWidth == 64)
2893 IID = Intrinsic::x86_avx_vpermilvar_pd;
2894 else if (VecWidth == 256 && EltWidth == 32)
2895 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2896 else if (VecWidth == 256 && EltWidth == 64)
2897 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2898 else if (VecWidth == 512 && EltWidth == 32)
2899 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2900 else if (VecWidth == 512 && EltWidth == 64)
2901 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2902 else
2903 llvm_unreachable("Unexpected intrinsic");
2904 } else if (Name == "cvtpd2dq.256") {
2905 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2906 } else if (Name == "cvtpd2ps.256") {
2907 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2908 } else if (Name == "cvttpd2dq.256") {
2909 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2910 } else if (Name == "cvttps2dq.128") {
2911 IID = Intrinsic::x86_sse2_cvttps2dq;
2912 } else if (Name == "cvttps2dq.256") {
2913 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2914 } else if (Name.starts_with("permvar.")) {
2915 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2916 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2917 IID = Intrinsic::x86_avx2_permps;
2918 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2919 IID = Intrinsic::x86_avx2_permd;
2920 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2921 IID = Intrinsic::x86_avx512_permvar_df_256;
2922 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2923 IID = Intrinsic::x86_avx512_permvar_di_256;
2924 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2925 IID = Intrinsic::x86_avx512_permvar_sf_512;
2926 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2927 IID = Intrinsic::x86_avx512_permvar_si_512;
2928 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2929 IID = Intrinsic::x86_avx512_permvar_df_512;
2930 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2931 IID = Intrinsic::x86_avx512_permvar_di_512;
2932 else if (VecWidth == 128 && EltWidth == 16)
2933 IID = Intrinsic::x86_avx512_permvar_hi_128;
2934 else if (VecWidth == 256 && EltWidth == 16)
2935 IID = Intrinsic::x86_avx512_permvar_hi_256;
2936 else if (VecWidth == 512 && EltWidth == 16)
2937 IID = Intrinsic::x86_avx512_permvar_hi_512;
2938 else if (VecWidth == 128 && EltWidth == 8)
2939 IID = Intrinsic::x86_avx512_permvar_qi_128;
2940 else if (VecWidth == 256 && EltWidth == 8)
2941 IID = Intrinsic::x86_avx512_permvar_qi_256;
2942 else if (VecWidth == 512 && EltWidth == 8)
2943 IID = Intrinsic::x86_avx512_permvar_qi_512;
2944 else
2945 llvm_unreachable("Unexpected intrinsic");
2946 } else if (Name.starts_with("dbpsadbw.")) {
2947 if (VecWidth == 128)
2948 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2949 else if (VecWidth == 256)
2950 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2951 else if (VecWidth == 512)
2952 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2953 else
2954 llvm_unreachable("Unexpected intrinsic");
2955 } else if (Name.starts_with("pmultishift.qb.")) {
2956 if (VecWidth == 128)
2957 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2958 else if (VecWidth == 256)
2959 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2960 else if (VecWidth == 512)
2961 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2962 else
2963 llvm_unreachable("Unexpected intrinsic");
2964 } else if (Name.starts_with("conflict.")) {
2965 if (Name[9] == 'd' && VecWidth == 128)
2966 IID = Intrinsic::x86_avx512_conflict_d_128;
2967 else if (Name[9] == 'd' && VecWidth == 256)
2968 IID = Intrinsic::x86_avx512_conflict_d_256;
2969 else if (Name[9] == 'd' && VecWidth == 512)
2970 IID = Intrinsic::x86_avx512_conflict_d_512;
2971 else if (Name[9] == 'q' && VecWidth == 128)
2972 IID = Intrinsic::x86_avx512_conflict_q_128;
2973 else if (Name[9] == 'q' && VecWidth == 256)
2974 IID = Intrinsic::x86_avx512_conflict_q_256;
2975 else if (Name[9] == 'q' && VecWidth == 512)
2976 IID = Intrinsic::x86_avx512_conflict_q_512;
2977 else
2978 llvm_unreachable("Unexpected intrinsic");
2979 } else if (Name.starts_with("pavg.")) {
2980 if (Name[5] == 'b' && VecWidth == 128)
2981 IID = Intrinsic::x86_sse2_pavg_b;
2982 else if (Name[5] == 'b' && VecWidth == 256)
2983 IID = Intrinsic::x86_avx2_pavg_b;
2984 else if (Name[5] == 'b' && VecWidth == 512)
2985 IID = Intrinsic::x86_avx512_pavg_b_512;
2986 else if (Name[5] == 'w' && VecWidth == 128)
2987 IID = Intrinsic::x86_sse2_pavg_w;
2988 else if (Name[5] == 'w' && VecWidth == 256)
2989 IID = Intrinsic::x86_avx2_pavg_w;
2990 else if (Name[5] == 'w' && VecWidth == 512)
2991 IID = Intrinsic::x86_avx512_pavg_w_512;
2992 else
2993 llvm_unreachable("Unexpected intrinsic");
2994 } else
2995 return false;
2996
2997 SmallVector<Value *, 4> Args(CI.args());
2998 Args.pop_back();
2999 Args.pop_back();
3000 Rep = Builder.CreateIntrinsic(IID, Args);
3001 unsigned NumArgs = CI.arg_size();
3002 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3003 CI.getArgOperand(NumArgs - 2));
3004 return true;
3005}
3006
3007/// Upgrade comment in call to inline asm that represents an objc retain release
3008/// marker.
3009void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3010 size_t Pos;
3011 if (AsmStr->find("mov\tfp") == 0 &&
3012 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3013 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3014 AsmStr->replace(Pos, 1, ";");
3015 }
3016}
3017
3018static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3019 Function *F, IRBuilder<> &Builder) {
3020 Value *Rep = nullptr;
3021
3022 if (Name == "abs.i" || Name == "abs.ll") {
3023 Value *Arg = CI->getArgOperand(0);
3024 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3025 {Arg, Builder.getTrue()},
3026 /*FMFSource=*/nullptr, "abs");
3027 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3028 Type *Ty = (Name == "abs.bf16")
3029 ? Builder.getBFloatTy()
3030 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3031 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3032 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3033 Rep = Builder.CreateBitCast(Abs, CI->getType());
3034 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3035 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3036 : Intrinsic::nvvm_fabs;
3037 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3038 } else if (Name.consume_front("ex2.approx.")) {
3039 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3040 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3041 : Intrinsic::nvvm_ex2_approx;
3042 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3043 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3044 Name.starts_with("atomic.load.add.f64.p")) {
3045 Value *Ptr = CI->getArgOperand(0);
3046 Value *Val = CI->getArgOperand(1);
3047 Rep = Builder.CreateAtomicRMW(
3048 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3049 CI->getContext().getOrInsertSyncScopeID("device"));
3050 // The default scope for atomic.load.* intrinsics is device
3051 // (= gpu scope in ptx), but the default LLVM atomic scope is
3052 // "system"
3053 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3054 Name.starts_with("atomic.load.dec.32.p")) {
3055 Value *Ptr = CI->getArgOperand(0);
3056 Value *Val = CI->getArgOperand(1);
3057 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3058 : AtomicRMWInst::UDecWrap;
3059 Rep = Builder.CreateAtomicRMW(
3060 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3061 CI->getContext().getOrInsertSyncScopeID("device"));
3062 // See comment above.
3063 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3064 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3065 StringRef Op = Name.substr(StringRef("atomic.").size());
3066 Value *Ptr = CI->getArgOperand(0);
3067 Value *Val = CI->getArgOperand(1);
3068 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3069 Op.contains(".cta.") ? "block" : "");
3070 if (Op.starts_with("cas.")) {
3071 Value *New = CI->getArgOperand(2);
3072 Value *Pair = Builder.CreateAtomicCmpXchg(
3073 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3074 AtomicOrdering::Monotonic, SSID);
3075 Rep = Builder.CreateExtractValue(Pair, 0);
3076 } else {
3077 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3078 // because we were actually missing those intrinsics!
3079 AtomicRMWInst::BinOp BinOp =
3080 StringSwitch<AtomicRMWInst::BinOp>(Op)
3081 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3082 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3083 .StartsWith("exch.", AtomicRMWInst::Xchg)
3084 .StartsWith("max.", AtomicRMWInst::Max)
3085 .StartsWith("min.", AtomicRMWInst::Min)
3086 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3087 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3088 .StartsWith("and.", AtomicRMWInst::And)
3089 .StartsWith("or.", AtomicRMWInst::Or)
3090 .StartsWith("xor.", AtomicRMWInst::Xor)
3091 .Default(AtomicRMWInst::BAD_BINOP);
3092 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3093 "unexpected nvvm scoped atomic intrinsic");
3094 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3095 AtomicOrdering::Monotonic, SSID);
3096 }
3097 } else if (Name == "clz.ll") {
3098 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3099 Value *Arg = CI->getArgOperand(0);
3100 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3101 {Arg, Builder.getFalse()},
3102 /*FMFSource=*/nullptr, "ctlz");
3103 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3104 } else if (Name == "popc.ll") {
3105 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3106 // i64.
3107 Value *Arg = CI->getArgOperand(0);
3108 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3109 Arg, /*FMFSource=*/nullptr, "ctpop");
3110 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3111 } else if (Name == "h2f") {
3112 Value *Cast =
3113 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3114 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3115 } else if (Name.consume_front("bitcast.") &&
3116 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3117 Name == "d2ll")) {
3118 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3119 } else if (Name == "rotate.b32") {
3120 Value *Arg = CI->getOperand(0);
3121 Value *ShiftAmt = CI->getOperand(1);
3122 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3123 {Arg, Arg, ShiftAmt});
3124 } else if (Name == "rotate.b64") {
3125 Type *Int64Ty = Builder.getInt64Ty();
3126 Value *Arg = CI->getOperand(0);
3127 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3128 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3129 {Arg, Arg, ZExtShiftAmt});
3130 } else if (Name == "rotate.right.b64") {
3131 Type *Int64Ty = Builder.getInt64Ty();
3132 Value *Arg = CI->getOperand(0);
3133 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3134 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3135 {Arg, Arg, ZExtShiftAmt});
3136 } else if (Name == "swap.lo.hi.b64") {
3137 Type *Int64Ty = Builder.getInt64Ty();
3138 Value *Arg = CI->getOperand(0);
3139 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3140 {Arg, Arg, Builder.getInt64(32)});
3141 } else if ((Name.consume_front("ptr.gen.to.") &&
3142 consumeNVVMPtrAddrSpace(Name)) ||
3143 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3144 Name.starts_with(".to.gen"))) {
3145 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3146 } else if (Name.consume_front("ldg.global")) {
3147 Value *Ptr = CI->getArgOperand(0);
3148 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3149 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3150 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3151 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3152 MDNode *MD = MDNode::get(Builder.getContext(), {});
3153 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3154 return LD;
3155 } else if (Name == "tanh.approx.f32") {
3156 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3157 FastMathFlags FMF;
3158 FMF.setApproxFunc();
3159 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3160 FMF);
3161 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3162 Value *Arg =
3163 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3164 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3165 {}, {Arg});
3166 } else if (Name == "barrier") {
3167 Rep = Builder.CreateIntrinsic(
3168 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3169 {CI->getArgOperand(0), CI->getArgOperand(1)});
3170 } else if (Name == "barrier.sync") {
3171 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3172 {CI->getArgOperand(0)});
3173 } else if (Name == "barrier.sync.cnt") {
3174 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3175 {CI->getArgOperand(0), CI->getArgOperand(1)});
3176 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3177 Name == "barrier0.or") {
3178 Value *C = CI->getArgOperand(0);
3179 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3180
3181 Intrinsic::ID IID =
3182 StringSwitch<Intrinsic::ID>(Name)
3183 .Case("barrier0.popc",
3184 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3185 .Case("barrier0.and",
3186 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3187 .Case("barrier0.or",
3188 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3189 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3190 Rep = Builder.CreateZExt(Bar, CI->getType());
3191 } else {
3192 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3193 if (IID != Intrinsic::not_intrinsic &&
3194 !F->getReturnType()->getScalarType()->isBFloatTy()) {
3195 rename(F);
3196 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3197 SmallVector<Value *, 2> Args;
3198 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3199 Value *Arg = CI->getArgOperand(I);
3200 Type *OldType = Arg->getType();
3201 Type *NewType = NewFn->getArg(I)->getType();
3202 Args.push_back(
3203 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3204 ? Builder.CreateBitCast(Arg, NewType)
3205 : Arg);
3206 }
3207 Rep = Builder.CreateCall(NewFn, Args);
3208 if (F->getReturnType()->isIntegerTy())
3209 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3210 }
3211 }
3212
3213 return Rep;
3214}
3215
3216static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3217 IRBuilder<> &Builder) {
3218 LLVMContext &C = F->getContext();
3219 Value *Rep = nullptr;
3220
3221 if (Name.starts_with("sse4a.movnt.")) {
3222 SmallVector<Metadata *, 1> Elts;
3223 Elts.push_back(
3224 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3225 MDNode *Node = MDNode::get(C, Elts);
3226
3227 Value *Arg0 = CI->getArgOperand(0);
3228 Value *Arg1 = CI->getArgOperand(1);
3229
3230 // Nontemporal (unaligned) store of the 0'th element of the float/double
3231 // vector.
3232 Value *Extract =
3233 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3234
3235 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3236 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3237 } else if (Name.starts_with("avx.movnt.") ||
3238 Name.starts_with("avx512.storent.")) {
3239 SmallVector<Metadata *, 1> Elts;
3240 Elts.push_back(
3241 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3242 MDNode *Node = MDNode::get(C, Elts);
3243
3244 Value *Arg0 = CI->getArgOperand(0);
3245 Value *Arg1 = CI->getArgOperand(1);
3246
3247 StoreInst *SI = Builder.CreateAlignedStore(
3248 Arg1, Arg0,
3249 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3250 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3251 } else if (Name == "sse2.storel.dq") {
3252 Value *Arg0 = CI->getArgOperand(0);
3253 Value *Arg1 = CI->getArgOperand(1);
3254
3255 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3256 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3257 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3258 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3259 } else if (Name.starts_with("sse.storeu.") ||
3260 Name.starts_with("sse2.storeu.") ||
3261 Name.starts_with("avx.storeu.")) {
3262 Value *Arg0 = CI->getArgOperand(0);
3263 Value *Arg1 = CI->getArgOperand(1);
3264 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3265 } else if (Name == "avx512.mask.store.ss") {
3266 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3267 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3268 Mask, false);
3269 } else if (Name.starts_with("avx512.mask.store")) {
3270 // "avx512.mask.storeu." or "avx512.mask.store."
3271 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3272 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3273 CI->getArgOperand(2), Aligned);
3274 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3275 // Upgrade packed integer vector compare intrinsics to compare instructions.
3276 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3277 bool CmpEq = Name[9] == 'e';
3278 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3279 CI->getArgOperand(0), CI->getArgOperand(1));
3280 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3281 } else if (Name.starts_with("avx512.broadcastm")) {
3282 Type *ExtTy = Type::getInt32Ty(C);
3283 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3284 ExtTy = Type::getInt64Ty(C);
3285 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3286 ExtTy->getPrimitiveSizeInBits();
3287 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3288 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3289 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3290 Value *Vec = CI->getArgOperand(0);
3291 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3292 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3293 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3294 } else if (Name.starts_with("avx.sqrt.p") ||
3295 Name.starts_with("sse2.sqrt.p") ||
3296 Name.starts_with("sse.sqrt.p")) {
3297 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3298 {CI->getArgOperand(0)});
3299 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3300 if (CI->arg_size() == 4 &&
3301 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3302 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3303 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3304 : Intrinsic::x86_avx512_sqrt_pd_512;
3305
3306 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3307 Rep = Builder.CreateIntrinsic(IID, Args);
3308 } else {
3309 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3310 {CI->getArgOperand(0)});
3311 }
3312 Rep =
3313 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3314 } else if (Name.starts_with("avx512.ptestm") ||
3315 Name.starts_with("avx512.ptestnm")) {
3316 Value *Op0 = CI->getArgOperand(0);
3317 Value *Op1 = CI->getArgOperand(1);
3318 Value *Mask = CI->getArgOperand(2);
3319 Rep = Builder.CreateAnd(Op0, Op1);
3320 llvm::Type *Ty = Op0->getType();
3321 Value *Zero = llvm::Constant::getNullValue(Ty);
3322 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3323 ? ICmpInst::ICMP_NE
3324 : ICmpInst::ICMP_EQ;
3325 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3326 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3327 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3328 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3329 ->getNumElements();
3330 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3331 Rep =
3332 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3333 } else if (Name.starts_with("avx512.kunpck")) {
3334 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3335 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3336 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3337 int Indices[64];
3338 for (unsigned i = 0; i != NumElts; ++i)
3339 Indices[i] = i;
3340
3341 // First extract half of each vector. This gives better codegen than
3342 // doing it in a single shuffle.
3343 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3344 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3345 // Concat the vectors.
3346 // NOTE: Operands have to be swapped to match intrinsic definition.
3347 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3348 Rep = Builder.CreateBitCast(Rep, CI->getType());
3349 } else if (Name == "avx512.kand.w") {
3350 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3351 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3352 Rep = Builder.CreateAnd(LHS, RHS);
3353 Rep = Builder.CreateBitCast(Rep, CI->getType());
3354 } else if (Name == "avx512.kandn.w") {
3355 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3356 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3357 LHS = Builder.CreateNot(LHS);
3358 Rep = Builder.CreateAnd(LHS, RHS);
3359 Rep = Builder.CreateBitCast(Rep, CI->getType());
3360 } else if (Name == "avx512.kor.w") {
3361 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3362 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3363 Rep = Builder.CreateOr(LHS, RHS);
3364 Rep = Builder.CreateBitCast(Rep, CI->getType());
3365 } else if (Name == "avx512.kxor.w") {
3366 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3367 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3368 Rep = Builder.CreateXor(LHS, RHS);
3369 Rep = Builder.CreateBitCast(Rep, CI->getType());
3370 } else if (Name == "avx512.kxnor.w") {
3371 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3372 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3373 LHS = Builder.CreateNot(LHS);
3374 Rep = Builder.CreateXor(LHS, RHS);
3375 Rep = Builder.CreateBitCast(Rep, CI->getType());
3376 } else if (Name == "avx512.knot.w") {
3377 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3378 Rep = Builder.CreateNot(Rep);
3379 Rep = Builder.CreateBitCast(Rep, CI->getType());
3380 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3381 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3382 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3383 Rep = Builder.CreateOr(LHS, RHS);
3384 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3385 Value *C;
3386 if (Name[14] == 'c')
3387 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3388 else
3389 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3390 Rep = Builder.CreateICmpEQ(Rep, C);
3391 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3392 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3393 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3394 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3395 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3396 Type *I32Ty = Type::getInt32Ty(C);
3397 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3398 ConstantInt::get(I32Ty, 0));
3399 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3400 ConstantInt::get(I32Ty, 0));
3401 Value *EltOp;
3402 if (Name.contains(".add."))
3403 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3404 else if (Name.contains(".sub."))
3405 EltOp = Builder.CreateFSub(Elt0, Elt1);
3406 else if (Name.contains(".mul."))
3407 EltOp = Builder.CreateFMul(Elt0, Elt1);
3408 else
3409 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3410 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3411 ConstantInt::get(I32Ty, 0));
3412 } else if (Name.starts_with("avx512.mask.pcmp")) {
3413 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3414 bool CmpEq = Name[16] == 'e';
3415 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3416 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3417 Type *OpTy = CI->getArgOperand(0)->getType();
3418 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3419 Intrinsic::ID IID;
3420 switch (VecWidth) {
3421 default:
3422 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3423 break;
3424 case 128:
3425 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3426 break;
3427 case 256:
3428 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3429 break;
3430 case 512:
3431 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3432 break;
3433 }
3434
3435 Rep =
3436 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3437 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3438 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3439 Type *OpTy = CI->getArgOperand(0)->getType();
3440 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3441 unsigned EltWidth = OpTy->getScalarSizeInBits();
3442 Intrinsic::ID IID;
3443 if (VecWidth == 128 && EltWidth == 32)
3444 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3445 else if (VecWidth == 256 && EltWidth == 32)
3446 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3447 else if (VecWidth == 512 && EltWidth == 32)
3448 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3449 else if (VecWidth == 128 && EltWidth == 64)
3450 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3451 else if (VecWidth == 256 && EltWidth == 64)
3452 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3453 else if (VecWidth == 512 && EltWidth == 64)
3454 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3455 else
3456 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3457
3458 Rep =
3459 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3460 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3461 } else if (Name.starts_with("avx512.cmp.p")) {
3462 SmallVector<Value *, 4> Args(CI->args());
3463 Type *OpTy = Args[0]->getType();
3464 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3465 unsigned EltWidth = OpTy->getScalarSizeInBits();
3466 Intrinsic::ID IID;
3467 if (VecWidth == 128 && EltWidth == 32)
3468 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3469 else if (VecWidth == 256 && EltWidth == 32)
3470 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3471 else if (VecWidth == 512 && EltWidth == 32)
3472 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3473 else if (VecWidth == 128 && EltWidth == 64)
3474 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3475 else if (VecWidth == 256 && EltWidth == 64)
3476 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3477 else if (VecWidth == 512 && EltWidth == 64)
3478 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3479 else
3480 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3481
3482 Value *Mask = Constant::getAllOnesValue(CI->getType());
3483 if (VecWidth == 512)
3484 std::swap(Mask, Args.back());
3485 Args.push_back(Mask);
3486
3487 Rep = Builder.CreateIntrinsic(IID, Args);
3488 } else if (Name.starts_with("avx512.mask.cmp.")) {
3489 // Integer compare intrinsics.
3490 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3491 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3492 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3493 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3494 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3495 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3496 Name.starts_with("avx512.cvtw2mask.") ||
3497 Name.starts_with("avx512.cvtd2mask.") ||
3498 Name.starts_with("avx512.cvtq2mask.")) {
3499 Value *Op = CI->getArgOperand(0);
3500 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3501 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3502 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3503 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3504 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3505 Name.starts_with("avx512.mask.pabs")) {
3506 Rep = upgradeAbs(Builder, *CI);
3507 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3508 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3509 Name.starts_with("avx512.mask.pmaxs")) {
3510 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3511 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3512 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3513 Name.starts_with("avx512.mask.pmaxu")) {
3514 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3515 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3516 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3517 Name.starts_with("avx512.mask.pmins")) {
3518 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3519 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3520 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3521 Name.starts_with("avx512.mask.pminu")) {
3522 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3523 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3524 Name == "avx512.pmulu.dq.512" ||
3525 Name.starts_with("avx512.mask.pmulu.dq.")) {
3526 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3527 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3528 Name == "avx512.pmul.dq.512" ||
3529 Name.starts_with("avx512.mask.pmul.dq.")) {
3530 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3531 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3532 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3533 Rep =
3534 Builder.CreateSIToFP(CI->getArgOperand(1),
3535 cast<VectorType>(CI->getType())->getElementType());
3536 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3537 } else if (Name == "avx512.cvtusi2sd") {
3538 Rep =
3539 Builder.CreateUIToFP(CI->getArgOperand(1),
3540 cast<VectorType>(CI->getType())->getElementType());
3541 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3542 } else if (Name == "sse2.cvtss2sd") {
3543 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3544 Rep = Builder.CreateFPExt(
3545 Rep, cast<VectorType>(CI->getType())->getElementType());
3546 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3547 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3548 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3549 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3550 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3551 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3552 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3553 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3554 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3555 Name == "avx512.mask.cvtqq2ps.256" ||
3556 Name == "avx512.mask.cvtqq2ps.512" ||
3557 Name == "avx512.mask.cvtuqq2ps.256" ||
3558 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3559 Name == "avx.cvt.ps2.pd.256" ||
3560 Name == "avx512.mask.cvtps2pd.128" ||
3561 Name == "avx512.mask.cvtps2pd.256") {
3562 auto *DstTy = cast<FixedVectorType>(CI->getType());
3563 Rep = CI->getArgOperand(0);
3564 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3565
3566 unsigned NumDstElts = DstTy->getNumElements();
3567 if (NumDstElts < SrcTy->getNumElements()) {
3568 assert(NumDstElts == 2 && "Unexpected vector size");
3569 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3570 }
3571
3572 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3573 bool IsUnsigned = Name.contains("cvtu");
3574 if (IsPS2PD)
3575 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3576 else if (CI->arg_size() == 4 &&
3577 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3578 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3579 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3580 : Intrinsic::x86_avx512_sitofp_round;
3581 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3582 {Rep, CI->getArgOperand(3)});
3583 } else {
3584 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3585 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3586 }
3587
3588 if (CI->arg_size() >= 3)
3589 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3590 CI->getArgOperand(1));
3591 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3592 Name.starts_with("vcvtph2ps.")) {
3593 auto *DstTy = cast<FixedVectorType>(CI->getType());
3594 Rep = CI->getArgOperand(0);
3595 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3596 unsigned NumDstElts = DstTy->getNumElements();
3597 if (NumDstElts != SrcTy->getNumElements()) {
3598 assert(NumDstElts == 4 && "Unexpected vector size");
3599 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3600 }
3601 Rep = Builder.CreateBitCast(
3602 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3603 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3604 if (CI->arg_size() >= 3)
3605 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3606 CI->getArgOperand(1));
3607 } else if (Name.starts_with("avx512.mask.load")) {
3608 // "avx512.mask.loadu." or "avx512.mask.load."
3609 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3610 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3611 CI->getArgOperand(2), Aligned);
3612 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3613 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3614 auto *PtrTy = CI->getOperand(0)->getType();
3615 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3616 ResultTy->getNumElements());
3617 Rep = Builder.CreateIntrinsic(
3618 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3619 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3620 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3621 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3622 auto *PtrTy = CI->getArgOperand(0)->getType();
3623 Value *MaskVec =
3624 getX86MaskVec(Builder, CI->getArgOperand(2),
3625 cast<FixedVectorType>(ResultTy)->getNumElements());
3626 Rep = Builder.CreateIntrinsic(
3627 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3628 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3629 } else if (Name.starts_with("avx512.mask.compress.") ||
3630 Name.starts_with("avx512.mask.expand.")) {
3631 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3632
3633 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3634 ResultTy->getNumElements());
3635
3636 bool IsCompress = Name[12] == 'c';
3637 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3638 : Intrinsic::x86_avx512_mask_expand;
3639 Rep = Builder.CreateIntrinsic(
3640 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3641 } else if (Name.starts_with("xop.vpcom")) {
3642 bool IsSigned;
3643 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3644 Name.ends_with("uq"))
3645 IsSigned = false;
3646 else if (Name.ends_with("b") || Name.ends_with("w") ||
3647 Name.ends_with("d") || Name.ends_with("q"))
3648 IsSigned = true;
3649 else
3650 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3651
3652 unsigned Imm;
3653 if (CI->arg_size() == 3) {
3654 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3655 } else {
3656 Name = Name.substr(9); // strip off "xop.vpcom"
3657 if (Name.starts_with("lt"))
3658 Imm = 0;
3659 else if (Name.starts_with("le"))
3660 Imm = 1;
3661 else if (Name.starts_with("gt"))
3662 Imm = 2;
3663 else if (Name.starts_with("ge"))
3664 Imm = 3;
3665 else if (Name.starts_with("eq"))
3666 Imm = 4;
3667 else if (Name.starts_with("ne"))
3668 Imm = 5;
3669 else if (Name.starts_with("false"))
3670 Imm = 6;
3671 else if (Name.starts_with("true"))
3672 Imm = 7;
3673 else
3674 llvm_unreachable("Unknown condition");
3675 }
3676
3677 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3678 } else if (Name.starts_with("xop.vpcmov")) {
3679 Value *Sel = CI->getArgOperand(2);
3680 Value *NotSel = Builder.CreateNot(Sel);
3681 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3682 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3683 Rep = Builder.CreateOr(Sel0, Sel1);
3684 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3685 Name.starts_with("avx512.mask.prol")) {
3686 Rep = upgradeX86Rotate(Builder, *CI, false);
3687 } else if (Name.starts_with("avx512.pror") ||
3688 Name.starts_with("avx512.mask.pror")) {
3689 Rep = upgradeX86Rotate(Builder, *CI, true);
3690 } else if (Name.starts_with("avx512.vpshld.") ||
3691 Name.starts_with("avx512.mask.vpshld") ||
3692 Name.starts_with("avx512.maskz.vpshld")) {
3693 bool ZeroMask = Name[11] == 'z';
3694 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3695 } else if (Name.starts_with("avx512.vpshrd.") ||
3696 Name.starts_with("avx512.mask.vpshrd") ||
3697 Name.starts_with("avx512.maskz.vpshrd")) {
3698 bool ZeroMask = Name[11] == 'z';
3699 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3700 } else if (Name == "sse42.crc32.64.8") {
3701 Value *Trunc0 =
3702 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3703 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3704 {Trunc0, CI->getArgOperand(1)});
3705 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3706 } else if (Name.starts_with("avx.vbroadcast.s") ||
3707 Name.starts_with("avx512.vbroadcast.s")) {
3708 // Replace broadcasts with a series of insertelements.
3709 auto *VecTy = cast<FixedVectorType>(CI->getType());
3710 Type *EltTy = VecTy->getElementType();
3711 unsigned EltNum = VecTy->getNumElements();
3712 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3713 Type *I32Ty = Type::getInt32Ty(C);
3714 Rep = PoisonValue::get(VecTy);
3715 for (unsigned I = 0; I < EltNum; ++I)
3716 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3717 } else if (Name.starts_with("sse41.pmovsx") ||
3718 Name.starts_with("sse41.pmovzx") ||
3719 Name.starts_with("avx2.pmovsx") ||
3720 Name.starts_with("avx2.pmovzx") ||
3721 Name.starts_with("avx512.mask.pmovsx") ||
3722 Name.starts_with("avx512.mask.pmovzx")) {
3723 auto *DstTy = cast<FixedVectorType>(CI->getType());
3724 unsigned NumDstElts = DstTy->getNumElements();
3725
3726 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3727 SmallVector<int, 8> ShuffleMask(NumDstElts);
3728 for (unsigned i = 0; i != NumDstElts; ++i)
3729 ShuffleMask[i] = i;
3730
3731 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3732
3733 bool DoSext = Name.contains("pmovsx");
3734 Rep =
3735 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3736 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3737 if (CI->arg_size() == 3)
3738 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3739 CI->getArgOperand(1));
3740 } else if (Name == "avx512.mask.pmov.qd.256" ||
3741 Name == "avx512.mask.pmov.qd.512" ||
3742 Name == "avx512.mask.pmov.wb.256" ||
3743 Name == "avx512.mask.pmov.wb.512") {
3744 Type *Ty = CI->getArgOperand(1)->getType();
3745 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3746 Rep =
3747 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3748 } else if (Name.starts_with("avx.vbroadcastf128") ||
3749 Name == "avx2.vbroadcasti128") {
3750 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3751 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3752 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3753 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3754 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3755 if (NumSrcElts == 2)
3756 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3757 else
3758 Rep = Builder.CreateShuffleVector(Load,
3759 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3760 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3761 Name.starts_with("avx512.mask.shuf.f")) {
3762 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3763 Type *VT = CI->getType();
3764 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3765 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3766 unsigned ControlBitsMask = NumLanes - 1;
3767 unsigned NumControlBits = NumLanes / 2;
3768 SmallVector<int, 8> ShuffleMask(0);
3769
3770 for (unsigned l = 0; l != NumLanes; ++l) {
3771 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3772 // We actually need the other source.
3773 if (l >= NumLanes / 2)
3774 LaneMask += NumLanes;
3775 for (unsigned i = 0; i != NumElementsInLane; ++i)
3776 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3777 }
3778 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3779 CI->getArgOperand(1), ShuffleMask);
3780 Rep =
3781 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3782 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3783 Name.starts_with("avx512.mask.broadcasti")) {
3784 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3785 ->getNumElements();
3786 unsigned NumDstElts =
3787 cast<FixedVectorType>(CI->getType())->getNumElements();
3788
3789 SmallVector<int, 8> ShuffleMask(NumDstElts);
3790 for (unsigned i = 0; i != NumDstElts; ++i)
3791 ShuffleMask[i] = i % NumSrcElts;
3792
3793 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3794 CI->getArgOperand(0), ShuffleMask);
3795 Rep =
3796 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3797 } else if (Name.starts_with("avx2.pbroadcast") ||
3798 Name.starts_with("avx2.vbroadcast") ||
3799 Name.starts_with("avx512.pbroadcast") ||
3800 Name.starts_with("avx512.mask.broadcast.s")) {
3801 // Replace vp?broadcasts with a vector shuffle.
3802 Value *Op = CI->getArgOperand(0);
3803 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3804 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3805 SmallVector<int, 8> M;
3806 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3807 Rep = Builder.CreateShuffleVector(Op, M);
3808
3809 if (CI->arg_size() == 3)
3810 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3811 CI->getArgOperand(1));
3812 } else if (Name.starts_with("sse2.padds.") ||
3813 Name.starts_with("avx2.padds.") ||
3814 Name.starts_with("avx512.padds.") ||
3815 Name.starts_with("avx512.mask.padds.")) {
3816 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3817 } else if (Name.starts_with("sse2.psubs.") ||
3818 Name.starts_with("avx2.psubs.") ||
3819 Name.starts_with("avx512.psubs.") ||
3820 Name.starts_with("avx512.mask.psubs.")) {
3821 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3822 } else if (Name.starts_with("sse2.paddus.") ||
3823 Name.starts_with("avx2.paddus.") ||
3824 Name.starts_with("avx512.mask.paddus.")) {
3825 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3826 } else if (Name.starts_with("sse2.psubus.") ||
3827 Name.starts_with("avx2.psubus.") ||
3828 Name.starts_with("avx512.mask.psubus.")) {
3829 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3830 } else if (Name.starts_with("avx512.mask.palignr.")) {
3831 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3832 CI->getArgOperand(1), CI->getArgOperand(2),
3833 CI->getArgOperand(3), CI->getArgOperand(4),
3834 false);
3835 } else if (Name.starts_with("avx512.mask.valign.")) {
3836 Rep = upgradeX86ALIGNIntrinsics(
3837 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3838 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3839 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3840 // 128/256-bit shift left specified in bits.
3841 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3842 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3843 Shift / 8); // Shift is in bits.
3844 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3845 // 128/256-bit shift right specified in bits.
3846 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3847 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3848 Shift / 8); // Shift is in bits.
3849 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3850 Name == "avx512.psll.dq.512") {
3851 // 128/256/512-bit shift left specified in bytes.
3852 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3853 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3854 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3855 Name == "avx512.psrl.dq.512") {
3856 // 128/256/512-bit shift right specified in bytes.
3857 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3858 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3859 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3860 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3861 Name.starts_with("avx2.pblendd.")) {
3862 Value *Op0 = CI->getArgOperand(0);
3863 Value *Op1 = CI->getArgOperand(1);
3864 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3865 auto *VecTy = cast<FixedVectorType>(CI->getType());
3866 unsigned NumElts = VecTy->getNumElements();
3867
3868 SmallVector<int, 16> Idxs(NumElts);
3869 for (unsigned i = 0; i != NumElts; ++i)
3870 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3871
3872 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3873 } else if (Name.starts_with("avx.vinsertf128.") ||
3874 Name == "avx2.vinserti128" ||
3875 Name.starts_with("avx512.mask.insert")) {
3876 Value *Op0 = CI->getArgOperand(0);
3877 Value *Op1 = CI->getArgOperand(1);
3878 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3879 unsigned DstNumElts =
3880 cast<FixedVectorType>(CI->getType())->getNumElements();
3881 unsigned SrcNumElts =
3882 cast<FixedVectorType>(Op1->getType())->getNumElements();
3883 unsigned Scale = DstNumElts / SrcNumElts;
3884
3885 // Mask off the high bits of the immediate value; hardware ignores those.
3886 Imm = Imm % Scale;
3887
3888 // Extend the second operand into a vector the size of the destination.
3889 SmallVector<int, 8> Idxs(DstNumElts);
3890 for (unsigned i = 0; i != SrcNumElts; ++i)
3891 Idxs[i] = i;
3892 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3893 Idxs[i] = SrcNumElts;
3894 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3895
3896 // Insert the second operand into the first operand.
3897
3898 // Note that there is no guarantee that instruction lowering will actually
3899 // produce a vinsertf128 instruction for the created shuffles. In
3900 // particular, the 0 immediate case involves no lane changes, so it can
3901 // be handled as a blend.
3902
3903 // Example of shuffle mask for 32-bit elements:
3904 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3905 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3906
3907 // First fill with identify mask.
3908 for (unsigned i = 0; i != DstNumElts; ++i)
3909 Idxs[i] = i;
3910 // Then replace the elements where we need to insert.
3911 for (unsigned i = 0; i != SrcNumElts; ++i)
3912 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3913 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3914
3915 // If the intrinsic has a mask operand, handle that.
3916 if (CI->arg_size() == 5)
3917 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3918 CI->getArgOperand(3));
3919 } else if (Name.starts_with("avx.vextractf128.") ||
3920 Name == "avx2.vextracti128" ||
3921 Name.starts_with("avx512.mask.vextract")) {
3922 Value *Op0 = CI->getArgOperand(0);
3923 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3924 unsigned DstNumElts =
3925 cast<FixedVectorType>(CI->getType())->getNumElements();
3926 unsigned SrcNumElts =
3927 cast<FixedVectorType>(Op0->getType())->getNumElements();
3928 unsigned Scale = SrcNumElts / DstNumElts;
3929
3930 // Mask off the high bits of the immediate value; hardware ignores those.
3931 Imm = Imm % Scale;
3932
3933 // Get indexes for the subvector of the input vector.
3934 SmallVector<int, 8> Idxs(DstNumElts);
3935 for (unsigned i = 0; i != DstNumElts; ++i) {
3936 Idxs[i] = i + (Imm * DstNumElts);
3937 }
3938 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3939
3940 // If the intrinsic has a mask operand, handle that.
3941 if (CI->arg_size() == 4)
3942 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3943 CI->getArgOperand(2));
3944 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3945 Name.starts_with("avx512.mask.perm.di.")) {
3946 Value *Op0 = CI->getArgOperand(0);
3947 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3948 auto *VecTy = cast<FixedVectorType>(CI->getType());
3949 unsigned NumElts = VecTy->getNumElements();
3950
3951 SmallVector<int, 8> Idxs(NumElts);
3952 for (unsigned i = 0; i != NumElts; ++i)
3953 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3954
3955 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3956
3957 if (CI->arg_size() == 4)
3958 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3959 CI->getArgOperand(2));
3960 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3961 // The immediate permute control byte looks like this:
3962 // [1:0] - select 128 bits from sources for low half of destination
3963 // [2] - ignore
3964 // [3] - zero low half of destination
3965 // [5:4] - select 128 bits from sources for high half of destination
3966 // [6] - ignore
3967 // [7] - zero high half of destination
3968
3969 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3970
3971 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3972 unsigned HalfSize = NumElts / 2;
3973 SmallVector<int, 8> ShuffleMask(NumElts);
3974
3975 // Determine which operand(s) are actually in use for this instruction.
3976 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3977 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3978
3979 // If needed, replace operands based on zero mask.
3980 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3981 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3982
3983 // Permute low half of result.
3984 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3985 for (unsigned i = 0; i < HalfSize; ++i)
3986 ShuffleMask[i] = StartIndex + i;
3987
3988 // Permute high half of result.
3989 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3990 for (unsigned i = 0; i < HalfSize; ++i)
3991 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3992
3993 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3994
3995 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
3996 Name.starts_with("avx512.mask.vpermil.p") ||
3997 Name.starts_with("avx512.mask.pshuf.d.")) {
3998 Value *Op0 = CI->getArgOperand(0);
3999 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4000 auto *VecTy = cast<FixedVectorType>(CI->getType());
4001 unsigned NumElts = VecTy->getNumElements();
4002 // Calculate the size of each index in the immediate.
4003 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4004 unsigned IdxMask = ((1 << IdxSize) - 1);
4005
4006 SmallVector<int, 8> Idxs(NumElts);
4007 // Lookup the bits for this element, wrapping around the immediate every
4008 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4009 // to offset by the first index of each group.
4010 for (unsigned i = 0; i != NumElts; ++i)
4011 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4012
4013 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4014
4015 if (CI->arg_size() == 4)
4016 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4017 CI->getArgOperand(2));
4018 } else if (Name == "sse2.pshufl.w" ||
4019 Name.starts_with("avx512.mask.pshufl.w.")) {
4020 Value *Op0 = CI->getArgOperand(0);
4021 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4022 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4023
4024 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4025 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4026
4027 SmallVector<int, 16> Idxs(NumElts);
4028 for (unsigned l = 0; l != NumElts; l += 8) {
4029 for (unsigned i = 0; i != 4; ++i)
4030 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4031 for (unsigned i = 4; i != 8; ++i)
4032 Idxs[i + l] = i + l;
4033 }
4034
4035 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4036
4037 if (CI->arg_size() == 4)
4038 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4039 CI->getArgOperand(2));
4040 } else if (Name == "sse2.pshufh.w" ||
4041 Name.starts_with("avx512.mask.pshufh.w.")) {
4042 Value *Op0 = CI->getArgOperand(0);
4043 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4044 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4045
4046 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4047 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4048
4049 SmallVector<int, 16> Idxs(NumElts);
4050 for (unsigned l = 0; l != NumElts; l += 8) {
4051 for (unsigned i = 0; i != 4; ++i)
4052 Idxs[i + l] = i + l;
4053 for (unsigned i = 0; i != 4; ++i)
4054 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4055 }
4056
4057 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4058
4059 if (CI->arg_size() == 4)
4060 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4061 CI->getArgOperand(2));
4062 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4063 Value *Op0 = CI->getArgOperand(0);
4064 Value *Op1 = CI->getArgOperand(1);
4065 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4066 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4067
4068 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4069 unsigned HalfLaneElts = NumLaneElts / 2;
4070
4071 SmallVector<int, 16> Idxs(NumElts);
4072 for (unsigned i = 0; i != NumElts; ++i) {
4073 // Base index is the starting element of the lane.
4074 Idxs[i] = i - (i % NumLaneElts);
4075 // If we are half way through the lane switch to the other source.
4076 if ((i % NumLaneElts) >= HalfLaneElts)
4077 Idxs[i] += NumElts;
4078 // Now select the specific element. By adding HalfLaneElts bits from
4079 // the immediate. Wrapping around the immediate every 8-bits.
4080 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4081 }
4082
4083 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4084
4085 Rep =
4086 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4087 } else if (Name.starts_with("avx512.mask.movddup") ||
4088 Name.starts_with("avx512.mask.movshdup") ||
4089 Name.starts_with("avx512.mask.movsldup")) {
4090 Value *Op0 = CI->getArgOperand(0);
4091 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4092 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4093
4094 unsigned Offset = 0;
4095 if (Name.starts_with("avx512.mask.movshdup."))
4096 Offset = 1;
4097
4098 SmallVector<int, 16> Idxs(NumElts);
4099 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4100 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4101 Idxs[i + l + 0] = i + l + Offset;
4102 Idxs[i + l + 1] = i + l + Offset;
4103 }
4104
4105 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4106
4107 Rep =
4108 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4109 } else if (Name.starts_with("avx512.mask.punpckl") ||
4110 Name.starts_with("avx512.mask.unpckl.")) {
4111 Value *Op0 = CI->getArgOperand(0);
4112 Value *Op1 = CI->getArgOperand(1);
4113 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4114 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4115
4116 SmallVector<int, 64> Idxs(NumElts);
4117 for (int l = 0; l != NumElts; l += NumLaneElts)
4118 for (int i = 0; i != NumLaneElts; ++i)
4119 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4120
4121 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4122
4123 Rep =
4124 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4125 } else if (Name.starts_with("avx512.mask.punpckh") ||
4126 Name.starts_with("avx512.mask.unpckh.")) {
4127 Value *Op0 = CI->getArgOperand(0);
4128 Value *Op1 = CI->getArgOperand(1);
4129 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4130 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4131
4132 SmallVector<int, 64> Idxs(NumElts);
4133 for (int l = 0; l != NumElts; l += NumLaneElts)
4134 for (int i = 0; i != NumLaneElts; ++i)
4135 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4136
4137 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4138
4139 Rep =
4140 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4141 } else if (Name.starts_with("avx512.mask.and.") ||
4142 Name.starts_with("avx512.mask.pand.")) {
4143 VectorType *FTy = cast<VectorType>(CI->getType());
4144 VectorType *ITy = VectorType::getInteger(FTy);
4145 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4146 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4147 Rep = Builder.CreateBitCast(Rep, FTy);
4148 Rep =
4149 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4150 } else if (Name.starts_with("avx512.mask.andn.") ||
4151 Name.starts_with("avx512.mask.pandn.")) {
4152 VectorType *FTy = cast<VectorType>(CI->getType());
4153 VectorType *ITy = VectorType::getInteger(FTy);
4154 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4155 Rep = Builder.CreateAnd(Rep,
4156 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4157 Rep = Builder.CreateBitCast(Rep, FTy);
4158 Rep =
4159 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4160 } else if (Name.starts_with("avx512.mask.or.") ||
4161 Name.starts_with("avx512.mask.por.")) {
4162 VectorType *FTy = cast<VectorType>(CI->getType());
4163 VectorType *ITy = VectorType::getInteger(FTy);
4164 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4165 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4166 Rep = Builder.CreateBitCast(Rep, FTy);
4167 Rep =
4168 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4169 } else if (Name.starts_with("avx512.mask.xor.") ||
4170 Name.starts_with("avx512.mask.pxor.")) {
4171 VectorType *FTy = cast<VectorType>(CI->getType());
4172 VectorType *ITy = VectorType::getInteger(FTy);
4173 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4174 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4175 Rep = Builder.CreateBitCast(Rep, FTy);
4176 Rep =
4177 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4178 } else if (Name.starts_with("avx512.mask.padd.")) {
4179 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4180 Rep =
4181 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4182 } else if (Name.starts_with("avx512.mask.psub.")) {
4183 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4184 Rep =
4185 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4186 } else if (Name.starts_with("avx512.mask.pmull.")) {
4187 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4188 Rep =
4189 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4190 } else if (Name.starts_with("avx512.mask.add.p")) {
4191 if (Name.ends_with(".512")) {
4192 Intrinsic::ID IID;
4193 if (Name[17] == 's')
4194 IID = Intrinsic::x86_avx512_add_ps_512;
4195 else
4196 IID = Intrinsic::x86_avx512_add_pd_512;
4197
4198 Rep = Builder.CreateIntrinsic(
4199 IID,
4200 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4201 } else {
4202 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4203 }
4204 Rep =
4205 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4206 } else if (Name.starts_with("avx512.mask.div.p")) {
4207 if (Name.ends_with(".512")) {
4208 Intrinsic::ID IID;
4209 if (Name[17] == 's')
4210 IID = Intrinsic::x86_avx512_div_ps_512;
4211 else
4212 IID = Intrinsic::x86_avx512_div_pd_512;
4213
4214 Rep = Builder.CreateIntrinsic(
4215 IID,
4216 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4217 } else {
4218 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4219 }
4220 Rep =
4221 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4222 } else if (Name.starts_with("avx512.mask.mul.p")) {
4223 if (Name.ends_with(".512")) {
4224 Intrinsic::ID IID;
4225 if (Name[17] == 's')
4226 IID = Intrinsic::x86_avx512_mul_ps_512;
4227 else
4228 IID = Intrinsic::x86_avx512_mul_pd_512;
4229
4230 Rep = Builder.CreateIntrinsic(
4231 IID,
4232 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4233 } else {
4234 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4235 }
4236 Rep =
4237 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4238 } else if (Name.starts_with("avx512.mask.sub.p")) {
4239 if (Name.ends_with(".512")) {
4240 Intrinsic::ID IID;
4241 if (Name[17] == 's')
4242 IID = Intrinsic::x86_avx512_sub_ps_512;
4243 else
4244 IID = Intrinsic::x86_avx512_sub_pd_512;
4245
4246 Rep = Builder.CreateIntrinsic(
4247 IID,
4248 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4249 } else {
4250 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4251 }
4252 Rep =
4253 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4254 } else if ((Name.starts_with("avx512.mask.max.p") ||
4255 Name.starts_with("avx512.mask.min.p")) &&
4256 Name.drop_front(18) == ".512") {
4257 bool IsDouble = Name[17] == 'd';
4258 bool IsMin = Name[13] == 'i';
4259 static const Intrinsic::ID MinMaxTbl[2][2] = {
4260 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4261 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4262 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4263
4264 Rep = Builder.CreateIntrinsic(
4265 IID,
4266 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4267 Rep =
4268 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4269 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4270 Rep =
4271 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4272 {CI->getArgOperand(0), Builder.getInt1(false)});
4273 Rep =
4274 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4275 } else if (Name.starts_with("avx512.mask.psll")) {
4276 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4277 bool IsVariable = Name[16] == 'v';
4278 char Size = Name[16] == '.' ? Name[17]
4279 : Name[17] == '.' ? Name[18]
4280 : Name[18] == '.' ? Name[19]
4281 : Name[20];
4282
4283 Intrinsic::ID IID;
4284 if (IsVariable && Name[17] != '.') {
4285 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4286 IID = Intrinsic::x86_avx2_psllv_q;
4287 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4288 IID = Intrinsic::x86_avx2_psllv_q_256;
4289 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4290 IID = Intrinsic::x86_avx2_psllv_d;
4291 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4292 IID = Intrinsic::x86_avx2_psllv_d_256;
4293 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4294 IID = Intrinsic::x86_avx512_psllv_w_128;
4295 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4296 IID = Intrinsic::x86_avx512_psllv_w_256;
4297 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4298 IID = Intrinsic::x86_avx512_psllv_w_512;
4299 else
4300 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4301 } else if (Name.ends_with(".128")) {
4302 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4303 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4304 : Intrinsic::x86_sse2_psll_d;
4305 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4306 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4307 : Intrinsic::x86_sse2_psll_q;
4308 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4309 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4310 : Intrinsic::x86_sse2_psll_w;
4311 else
4312 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4313 } else if (Name.ends_with(".256")) {
4314 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4315 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4316 : Intrinsic::x86_avx2_psll_d;
4317 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4318 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4319 : Intrinsic::x86_avx2_psll_q;
4320 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4321 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4322 : Intrinsic::x86_avx2_psll_w;
4323 else
4324 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4325 } else {
4326 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4327 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4328 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4329 : Intrinsic::x86_avx512_psll_d_512;
4330 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4331 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4332 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4333 : Intrinsic::x86_avx512_psll_q_512;
4334 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4335 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4336 : Intrinsic::x86_avx512_psll_w_512;
4337 else
4338 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4339 }
4340
4341 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4342 } else if (Name.starts_with("avx512.mask.psrl")) {
4343 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4344 bool IsVariable = Name[16] == 'v';
4345 char Size = Name[16] == '.' ? Name[17]
4346 : Name[17] == '.' ? Name[18]
4347 : Name[18] == '.' ? Name[19]
4348 : Name[20];
4349
4350 Intrinsic::ID IID;
4351 if (IsVariable && Name[17] != '.') {
4352 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4353 IID = Intrinsic::x86_avx2_psrlv_q;
4354 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4355 IID = Intrinsic::x86_avx2_psrlv_q_256;
4356 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4357 IID = Intrinsic::x86_avx2_psrlv_d;
4358 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4359 IID = Intrinsic::x86_avx2_psrlv_d_256;
4360 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4361 IID = Intrinsic::x86_avx512_psrlv_w_128;
4362 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4363 IID = Intrinsic::x86_avx512_psrlv_w_256;
4364 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4365 IID = Intrinsic::x86_avx512_psrlv_w_512;
4366 else
4367 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4368 } else if (Name.ends_with(".128")) {
4369 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4370 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4371 : Intrinsic::x86_sse2_psrl_d;
4372 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4373 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4374 : Intrinsic::x86_sse2_psrl_q;
4375 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4376 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4377 : Intrinsic::x86_sse2_psrl_w;
4378 else
4379 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4380 } else if (Name.ends_with(".256")) {
4381 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4382 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4383 : Intrinsic::x86_avx2_psrl_d;
4384 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4385 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4386 : Intrinsic::x86_avx2_psrl_q;
4387 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4388 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4389 : Intrinsic::x86_avx2_psrl_w;
4390 else
4391 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4392 } else {
4393 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4394 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4395 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4396 : Intrinsic::x86_avx512_psrl_d_512;
4397 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4398 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4399 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4400 : Intrinsic::x86_avx512_psrl_q_512;
4401 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4402 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4403 : Intrinsic::x86_avx512_psrl_w_512;
4404 else
4405 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4406 }
4407
4408 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4409 } else if (Name.starts_with("avx512.mask.psra")) {
4410 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4411 bool IsVariable = Name[16] == 'v';
4412 char Size = Name[16] == '.' ? Name[17]
4413 : Name[17] == '.' ? Name[18]
4414 : Name[18] == '.' ? Name[19]
4415 : Name[20];
4416
4417 Intrinsic::ID IID;
4418 if (IsVariable && Name[17] != '.') {
4419 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4420 IID = Intrinsic::x86_avx2_psrav_d;
4421 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4422 IID = Intrinsic::x86_avx2_psrav_d_256;
4423 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4424 IID = Intrinsic::x86_avx512_psrav_w_128;
4425 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4426 IID = Intrinsic::x86_avx512_psrav_w_256;
4427 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4428 IID = Intrinsic::x86_avx512_psrav_w_512;
4429 else
4430 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4431 } else if (Name.ends_with(".128")) {
4432 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4433 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4434 : Intrinsic::x86_sse2_psra_d;
4435 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4436 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4437 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4438 : Intrinsic::x86_avx512_psra_q_128;
4439 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4440 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4441 : Intrinsic::x86_sse2_psra_w;
4442 else
4443 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4444 } else if (Name.ends_with(".256")) {
4445 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4446 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4447 : Intrinsic::x86_avx2_psra_d;
4448 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4449 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4450 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4451 : Intrinsic::x86_avx512_psra_q_256;
4452 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4453 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4454 : Intrinsic::x86_avx2_psra_w;
4455 else
4456 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4457 } else {
4458 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4459 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4460 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4461 : Intrinsic::x86_avx512_psra_d_512;
4462 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4463 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4464 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4465 : Intrinsic::x86_avx512_psra_q_512;
4466 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4467 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4468 : Intrinsic::x86_avx512_psra_w_512;
4469 else
4470 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4471 }
4472
4473 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4474 } else if (Name.starts_with("avx512.mask.move.s")) {
4475 Rep = upgradeMaskedMove(Builder, *CI);
4476 } else if (Name.starts_with("avx512.cvtmask2")) {
4477 Rep = upgradeMaskToInt(Builder, *CI);
4478 } else if (Name.ends_with(".movntdqa")) {
4479 MDNode *Node = MDNode::get(
4480 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4481
4482 LoadInst *LI = Builder.CreateAlignedLoad(
4483 CI->getType(), CI->getArgOperand(0),
4484 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4485 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4486 Rep = LI;
4487 } else if (Name.starts_with("fma.vfmadd.") ||
4488 Name.starts_with("fma.vfmsub.") ||
4489 Name.starts_with("fma.vfnmadd.") ||
4490 Name.starts_with("fma.vfnmsub.")) {
4491 bool NegMul = Name[6] == 'n';
4492 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4493 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4494
4495 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4496 CI->getArgOperand(2)};
4497
4498 if (IsScalar) {
4499 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4500 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4501 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4502 }
4503
4504 if (NegMul && !IsScalar)
4505 Ops[0] = Builder.CreateFNeg(Ops[0]);
4506 if (NegMul && IsScalar)
4507 Ops[1] = Builder.CreateFNeg(Ops[1]);
4508 if (NegAcc)
4509 Ops[2] = Builder.CreateFNeg(Ops[2]);
4510
4511 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4512
4513 if (IsScalar)
4514 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4515 } else if (Name.starts_with("fma4.vfmadd.s")) {
4516 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4517 CI->getArgOperand(2)};
4518
4519 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4520 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4521 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4522
4523 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4524
4525 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4526 Rep, (uint64_t)0);
4527 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4528 Name.starts_with("avx512.maskz.vfmadd.s") ||
4529 Name.starts_with("avx512.mask3.vfmadd.s") ||
4530 Name.starts_with("avx512.mask3.vfmsub.s") ||
4531 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4532 bool IsMask3 = Name[11] == '3';
4533 bool IsMaskZ = Name[11] == 'z';
4534 // Drop the "avx512.mask." to make it easier.
4535 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4536 bool NegMul = Name[2] == 'n';
4537 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4538
4539 Value *A = CI->getArgOperand(0);
4540 Value *B = CI->getArgOperand(1);
4541 Value *C = CI->getArgOperand(2);
4542
4543 if (NegMul && (IsMask3 || IsMaskZ))
4544 A = Builder.CreateFNeg(A);
4545 if (NegMul && !(IsMask3 || IsMaskZ))
4546 B = Builder.CreateFNeg(B);
4547 if (NegAcc)
4548 C = Builder.CreateFNeg(C);
4549
4550 A = Builder.CreateExtractElement(A, (uint64_t)0);
4551 B = Builder.CreateExtractElement(B, (uint64_t)0);
4552 C = Builder.CreateExtractElement(C, (uint64_t)0);
4553
4554 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4555 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4556 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4557
4558 Intrinsic::ID IID;
4559 if (Name.back() == 'd')
4560 IID = Intrinsic::x86_avx512_vfmadd_f64;
4561 else
4562 IID = Intrinsic::x86_avx512_vfmadd_f32;
4563 Rep = Builder.CreateIntrinsic(IID, Ops);
4564 } else {
4565 Rep = Builder.CreateFMA(A, B, C);
4566 }
4567
4568 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4569 : IsMask3 ? C
4570 : A;
4571
4572 // For Mask3 with NegAcc, we need to create a new extractelement that
4573 // avoids the negation above.
4574 if (NegAcc && IsMask3)
4575 PassThru =
4576 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4577
4578 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4579 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4580 (uint64_t)0);
4581 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4582 Name.starts_with("avx512.mask.vfnmadd.p") ||
4583 Name.starts_with("avx512.mask.vfnmsub.p") ||
4584 Name.starts_with("avx512.mask3.vfmadd.p") ||
4585 Name.starts_with("avx512.mask3.vfmsub.p") ||
4586 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4587 Name.starts_with("avx512.maskz.vfmadd.p")) {
4588 bool IsMask3 = Name[11] == '3';
4589 bool IsMaskZ = Name[11] == 'z';
4590 // Drop the "avx512.mask." to make it easier.
4591 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4592 bool NegMul = Name[2] == 'n';
4593 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4594
4595 Value *A = CI->getArgOperand(0);
4596 Value *B = CI->getArgOperand(1);
4597 Value *C = CI->getArgOperand(2);
4598
4599 if (NegMul && (IsMask3 || IsMaskZ))
4600 A = Builder.CreateFNeg(A);
4601 if (NegMul && !(IsMask3 || IsMaskZ))
4602 B = Builder.CreateFNeg(B);
4603 if (NegAcc)
4604 C = Builder.CreateFNeg(C);
4605
4606 if (CI->arg_size() == 5 &&
4607 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4608 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4609 Intrinsic::ID IID;
4610 // Check the character before ".512" in string.
4611 if (Name[Name.size() - 5] == 's')
4612 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4613 else
4614 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4615
4616 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4617 } else {
4618 Rep = Builder.CreateFMA(A, B, C);
4619 }
4620
4621 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4622 : IsMask3 ? CI->getArgOperand(2)
4623 : CI->getArgOperand(0);
4624
4625 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4626 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4627 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4628 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4629 Intrinsic::ID IID;
4630 if (VecWidth == 128 && EltWidth == 32)
4631 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4632 else if (VecWidth == 256 && EltWidth == 32)
4633 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4634 else if (VecWidth == 128 && EltWidth == 64)
4635 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4636 else if (VecWidth == 256 && EltWidth == 64)
4637 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4638 else
4639 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4640
4641 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4642 CI->getArgOperand(2)};
4643 Ops[2] = Builder.CreateFNeg(Ops[2]);
4644 Rep = Builder.CreateIntrinsic(IID, Ops);
4645 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4646 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4647 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4648 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4649 bool IsMask3 = Name[11] == '3';
4650 bool IsMaskZ = Name[11] == 'z';
4651 // Drop the "avx512.mask." to make it easier.
4652 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4653 bool IsSubAdd = Name[3] == 's';
4654 if (CI->arg_size() == 5) {
4655 Intrinsic::ID IID;
4656 // Check the character before ".512" in string.
4657 if (Name[Name.size() - 5] == 's')
4658 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4659 else
4660 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4661
4662 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4663 CI->getArgOperand(2), CI->getArgOperand(4)};
4664 if (IsSubAdd)
4665 Ops[2] = Builder.CreateFNeg(Ops[2]);
4666
4667 Rep = Builder.CreateIntrinsic(IID, Ops);
4668 } else {
4669 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4670
4671 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4672 CI->getArgOperand(2)};
4673
4674 Function *FMA = Intrinsic::getOrInsertDeclaration(
4675 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4676 Value *Odd = Builder.CreateCall(FMA, Ops);
4677 Ops[2] = Builder.CreateFNeg(Ops[2]);
4678 Value *Even = Builder.CreateCall(FMA, Ops);
4679
4680 if (IsSubAdd)
4681 std::swap(Even, Odd);
4682
4683 SmallVector<int, 32> Idxs(NumElts);
4684 for (int i = 0; i != NumElts; ++i)
4685 Idxs[i] = i + (i % 2) * NumElts;
4686
4687 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4688 }
4689
4690 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4691 : IsMask3 ? CI->getArgOperand(2)
4692 : CI->getArgOperand(0);
4693
4694 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4695 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4696 Name.starts_with("avx512.maskz.pternlog.")) {
4697 bool ZeroMask = Name[11] == 'z';
4698 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4699 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4700 Intrinsic::ID IID;
4701 if (VecWidth == 128 && EltWidth == 32)
4702 IID = Intrinsic::x86_avx512_pternlog_d_128;
4703 else if (VecWidth == 256 && EltWidth == 32)
4704 IID = Intrinsic::x86_avx512_pternlog_d_256;
4705 else if (VecWidth == 512 && EltWidth == 32)
4706 IID = Intrinsic::x86_avx512_pternlog_d_512;
4707 else if (VecWidth == 128 && EltWidth == 64)
4708 IID = Intrinsic::x86_avx512_pternlog_q_128;
4709 else if (VecWidth == 256 && EltWidth == 64)
4710 IID = Intrinsic::x86_avx512_pternlog_q_256;
4711 else if (VecWidth == 512 && EltWidth == 64)
4712 IID = Intrinsic::x86_avx512_pternlog_q_512;
4713 else
4714 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4715
4716 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4717 CI->getArgOperand(2), CI->getArgOperand(3)};
4718 Rep = Builder.CreateIntrinsic(IID, Args);
4719 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4720 : CI->getArgOperand(0);
4721 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4722 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4723 Name.starts_with("avx512.maskz.vpmadd52")) {
4724 bool ZeroMask = Name[11] == 'z';
4725 bool High = Name[20] == 'h' || Name[21] == 'h';
4726 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4727 Intrinsic::ID IID;
4728 if (VecWidth == 128 && !High)
4729 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4730 else if (VecWidth == 256 && !High)
4731 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4732 else if (VecWidth == 512 && !High)
4733 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4734 else if (VecWidth == 128 && High)
4735 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4736 else if (VecWidth == 256 && High)
4737 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4738 else if (VecWidth == 512 && High)
4739 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4740 else
4741 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4742
4743 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4744 CI->getArgOperand(2)};
4745 Rep = Builder.CreateIntrinsic(IID, Args);
4746 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4747 : CI->getArgOperand(0);
4748 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4749 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4750 Name.starts_with("avx512.mask.vpermt2var.") ||
4751 Name.starts_with("avx512.maskz.vpermt2var.")) {
4752 bool ZeroMask = Name[11] == 'z';
4753 bool IndexForm = Name[17] == 'i';
4754 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4755 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4756 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4757 Name.starts_with("avx512.mask.vpdpbusds.") ||
4758 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4759 bool ZeroMask = Name[11] == 'z';
4760 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4761 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4762 Intrinsic::ID IID;
4763 if (VecWidth == 128 && !IsSaturating)
4764 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4765 else if (VecWidth == 256 && !IsSaturating)
4766 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4767 else if (VecWidth == 512 && !IsSaturating)
4768 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4769 else if (VecWidth == 128 && IsSaturating)
4770 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4771 else if (VecWidth == 256 && IsSaturating)
4772 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4773 else if (VecWidth == 512 && IsSaturating)
4774 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4775 else
4776 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4777
4778 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4779 CI->getArgOperand(2)};
4780
4781 // Input arguments types were incorrectly set to vectors of i32 before but
4782 // they should be vectors of i8. Insert bit cast when encountering the old
4783 // types
4784 if (Args[1]->getType()->isVectorTy() &&
4785 cast<VectorType>(Args[1]->getType())
4786 ->getElementType()
4787 ->isIntegerTy(32) &&
4788 Args[2]->getType()->isVectorTy() &&
4789 cast<VectorType>(Args[2]->getType())
4790 ->getElementType()
4791 ->isIntegerTy(32)) {
4792 Type *NewArgType = nullptr;
4793 if (VecWidth == 128)
4794 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4795 else if (VecWidth == 256)
4796 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4797 else if (VecWidth == 512)
4798 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4799 else
4800 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4801 CI);
4802
4803 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4804 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4805 }
4806
4807 Rep = Builder.CreateIntrinsic(IID, Args);
4808 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4809 : CI->getArgOperand(0);
4810 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4811 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4812 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4813 Name.starts_with("avx512.mask.vpdpwssds.") ||
4814 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4815 bool ZeroMask = Name[11] == 'z';
4816 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4817 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4818 Intrinsic::ID IID;
4819 if (VecWidth == 128 && !IsSaturating)
4820 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4821 else if (VecWidth == 256 && !IsSaturating)
4822 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4823 else if (VecWidth == 512 && !IsSaturating)
4824 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4825 else if (VecWidth == 128 && IsSaturating)
4826 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4827 else if (VecWidth == 256 && IsSaturating)
4828 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4829 else if (VecWidth == 512 && IsSaturating)
4830 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4831 else
4832 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4833
4834 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4835 CI->getArgOperand(2)};
4836
4837 // Input arguments types were incorrectly set to vectors of i32 before but
4838 // they should be vectors of i16. Insert bit cast when encountering the old
4839 // types
4840 if (Args[1]->getType()->isVectorTy() &&
4841 cast<VectorType>(Args[1]->getType())
4842 ->getElementType()
4843 ->isIntegerTy(32) &&
4844 Args[2]->getType()->isVectorTy() &&
4845 cast<VectorType>(Args[2]->getType())
4846 ->getElementType()
4847 ->isIntegerTy(32)) {
4848 Type *NewArgType = nullptr;
4849 if (VecWidth == 128)
4850 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4851 else if (VecWidth == 256)
4852 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4853 else if (VecWidth == 512)
4854 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4855 else
4856 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4857 CI);
4858
4859 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4860 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4861 }
4862
4863 Rep = Builder.CreateIntrinsic(IID, Args);
4864 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4865 : CI->getArgOperand(0);
4866 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4867 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4868 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4869 Name == "subborrow.u32" || Name == "subborrow.u64") {
4870 Intrinsic::ID IID;
4871 if (Name[0] == 'a' && Name.back() == '2')
4872 IID = Intrinsic::x86_addcarry_32;
4873 else if (Name[0] == 'a' && Name.back() == '4')
4874 IID = Intrinsic::x86_addcarry_64;
4875 else if (Name[0] == 's' && Name.back() == '2')
4876 IID = Intrinsic::x86_subborrow_32;
4877 else if (Name[0] == 's' && Name.back() == '4')
4878 IID = Intrinsic::x86_subborrow_64;
4879 else
4880 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4881
4882 // Make a call with 3 operands.
4883 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4884 CI->getArgOperand(2)};
4885 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4886
4887 // Extract the second result and store it.
4888 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4889 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4890 // Replace the original call result with the first result of the new call.
4891 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4892
4893 CI->replaceAllUsesWith(CF);
4894 Rep = nullptr;
4895 } else if (Name.starts_with("avx512.mask.") &&
4896 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4897 // Rep will be updated by the call in the condition.
4898 } else if (Name.starts_with("bmi.pdep.")) {
4899 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
4900 } else if (Name.starts_with("bmi.pext.")) {
4901 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
4902 } else
4903 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4904
4905 return Rep;
4906}
4907
4908static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4909 Function *F, IRBuilder<> &Builder) {
4910 if (Name.starts_with("neon.bfcvt")) {
4911 if (Name.starts_with("neon.bfcvtn2")) {
4912 SmallVector<int, 32> LoMask(4);
4913 std::iota(LoMask.begin(), LoMask.end(), 0);
4914 SmallVector<int, 32> ConcatMask(8);
4915 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4916 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4917 Value *Trunc =
4918 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4919 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4920 } else if (Name.starts_with("neon.bfcvtn")) {
4921 SmallVector<int, 32> ConcatMask(8);
4922 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4923 Type *V4BF16 =
4924 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4925 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4926 dbgs() << "Trunc: " << *Trunc << "\n";
4927 return Builder.CreateShuffleVector(
4928 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4929 } else {
4930 return Builder.CreateFPTrunc(CI->getOperand(0),
4931 Type::getBFloatTy(F->getContext()));
4932 }
4933 } else if (Name.starts_with("sve.fcvt")) {
4934 Intrinsic::ID NewID =
4935 StringSwitch<Intrinsic::ID>(Name)
4936 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4937 .Case("sve.fcvtnt.bf16f32",
4938 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4939 .Default(Intrinsic::not_intrinsic);
4940 if (NewID == Intrinsic::not_intrinsic)
4941 llvm_unreachable("Unhandled Intrinsic!");
4942
4943 SmallVector<Value *, 3> Args(CI->args());
4944
4945 // The original intrinsics incorrectly used a predicate based on the
4946 // smallest element type rather than the largest.
4947 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4948 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4949
4950 if (Args[1]->getType() != BadPredTy)
4951 llvm_unreachable("Unexpected predicate type!");
4952
4953 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4954 BadPredTy, Args[1]);
4955 Args[1] = Builder.CreateIntrinsic(
4956 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4957
4958 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
4959 CI->getName());
4960 }
4961
4962 if (Name == "neon.vcvtfp2hf")
4963 return Builder.CreateBitCast(
4964 Builder.CreateFPTrunc(
4965 CI->getOperand(0),
4966 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4967 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
4968 if (Name == "neon.vcvthf2fp")
4969 return Builder.CreateFPExt(
4970 Builder.CreateBitCast(
4971 CI->getOperand(0),
4972 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4973 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
4974
4975 llvm_unreachable("Unhandled Intrinsic!");
4976}
4977
4978static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
4979 IRBuilder<> &Builder) {
4980 if (Name == "mve.vctp64.old") {
4981 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
4982 // correct type.
4983 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
4984 CI->getArgOperand(0),
4985 /*FMFSource=*/nullptr, CI->getName());
4986 Value *C1 = Builder.CreateIntrinsic(
4987 Intrinsic::arm_mve_pred_v2i,
4988 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
4989 return Builder.CreateIntrinsic(
4990 Intrinsic::arm_mve_pred_i2v,
4991 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
4992 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4993 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4994 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4995 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4996 Name ==
4997 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4998 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4999 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5000 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5001 Name ==
5002 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5003 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5004 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5005 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5006 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5007 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5008 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5009 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5010 std::vector<Type *> Tys;
5011 unsigned ID = CI->getIntrinsicID();
5012 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5013 switch (ID) {
5014 case Intrinsic::arm_mve_mull_int_predicated:
5015 case Intrinsic::arm_mve_vqdmull_predicated:
5016 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5017 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5018 break;
5019 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5020 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5021 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5022 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5023 V2I1Ty};
5024 break;
5025 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5026 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5027 CI->getOperand(1)->getType(), V2I1Ty};
5028 break;
5029 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5030 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5031 CI->getOperand(2)->getType(), V2I1Ty};
5032 break;
5033 case Intrinsic::arm_cde_vcx1q_predicated:
5034 case Intrinsic::arm_cde_vcx1qa_predicated:
5035 case Intrinsic::arm_cde_vcx2q_predicated:
5036 case Intrinsic::arm_cde_vcx2qa_predicated:
5037 case Intrinsic::arm_cde_vcx3q_predicated:
5038 case Intrinsic::arm_cde_vcx3qa_predicated:
5039 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5040 break;
5041 default:
5042 llvm_unreachable("Unhandled Intrinsic!");
5043 }
5044
5045 std::vector<Value *> Ops;
5046 for (Value *Op : CI->args()) {
5047 Type *Ty = Op->getType();
5048 if (Ty->getScalarSizeInBits() == 1) {
5049 Value *C1 = Builder.CreateIntrinsic(
5050 Intrinsic::arm_mve_pred_v2i,
5051 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5052 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5053 }
5054 Ops.push_back(Op);
5055 }
5056
5057 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5058 CI->getName());
5059 }
5060 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5061}
5062
5063// These are expected to have the arguments:
5064// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5065//
5066// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5067//
5068static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5069 Function *F, IRBuilder<> &Builder) {
5070 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5071 // for compatibility.
5072 auto UpgradeLegacyWMMAIUIntrinsicCall =
5073 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5074 ArrayRef<Type *> OverloadTys) -> Value * {
5075 // Prepare arguments, append clamp=0 for compatibility
5076 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5077 Args.push_back(Builder.getFalse());
5078
5079 // Insert the declaration for the right overload types
5080 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5081 F->getParent(), F->getIntrinsicID(), OverloadTys);
5082
5083 // Copy operand bundles if any
5084 SmallVector<OperandBundleDef, 1> Bundles;
5085 CI->getOperandBundlesAsDefs(Bundles);
5086
5087 // Create the new call and copy calling properties
5088 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5089 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5090 NewCall->setCallingConv(CI->getCallingConv());
5091 NewCall->setAttributes(CI->getAttributes());
5092 NewCall->setDebugLoc(CI->getDebugLoc());
5093 NewCall->copyMetadata(*CI);
5094 return NewCall;
5095 };
5096
5097 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5098 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5099 "intrinsic should have 7 arguments");
5100 Type *T1 = CI->getArgOperand(4)->getType();
5101 Type *T2 = CI->getArgOperand(1)->getType();
5102 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5103 }
5104 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5105 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5106 "intrinsic should have 8 arguments");
5107 Type *T1 = CI->getArgOperand(4)->getType();
5108 Type *T2 = CI->getArgOperand(1)->getType();
5109 Type *T3 = CI->getArgOperand(3)->getType();
5110 Type *T4 = CI->getArgOperand(5)->getType();
5111 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5112 }
5113
5114 switch (F->getIntrinsicID()) {
5115 default:
5116 break;
5117 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5118 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5119 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5120 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5121 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5122 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5123 // Drop src0 and src1 modifiers.
5124 const Value *Op0 = CI->getArgOperand(0);
5125 const Value *Op2 = CI->getArgOperand(2);
5126 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5127 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5128 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5129 if (!ModA->isZero() || !ModB->isZero())
5130 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5131
5132 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5133 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5134 Args.push_back(CI->getArgOperand(I));
5135
5136 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5137 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5138 Overloads.push_back(Args[3]->getType());
5139 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5140 F->getParent(), F->getIntrinsicID(), Overloads);
5141
5142 SmallVector<OperandBundleDef, 1> Bundles;
5143 CI->getOperandBundlesAsDefs(Bundles);
5144
5145 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5146 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5147 NewCall->setCallingConv(CI->getCallingConv());
5148 NewCall->setAttributes(CI->getAttributes());
5149 NewCall->setDebugLoc(CI->getDebugLoc());
5150 NewCall->copyMetadata(*CI);
5151 NewCall->takeName(CI);
5152 return NewCall;
5153 }
5154 }
5155
5156 AtomicRMWInst::BinOp RMWOp =
5157 StringSwitch<AtomicRMWInst::BinOp>(Name)
5158 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5159 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5160 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5161 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5162 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5163 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5164 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5165 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5166 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5167 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5168 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5169 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5170 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5171
5172 unsigned NumOperands = CI->getNumOperands();
5173 if (NumOperands < 3) // Malformed bitcode.
5174 return nullptr;
5175
5176 Value *Ptr = CI->getArgOperand(0);
5177 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5178 if (!PtrTy) // Malformed.
5179 return nullptr;
5180
5181 Value *Val = CI->getArgOperand(1);
5182 if (Val->getType() != CI->getType()) // Malformed.
5183 return nullptr;
5184
5185 ConstantInt *OrderArg = nullptr;
5186 bool IsVolatile = false;
5187
5188 // These should have 5 arguments (plus the callee). A separate version of the
5189 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5190 if (NumOperands > 3)
5191 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5192
5193 // Ignore scope argument at 3
5194
5195 if (NumOperands > 5) {
5196 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5197 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5198 }
5199
5200 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5201 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5202 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5203 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5204 Order = AtomicOrdering::SequentiallyConsistent;
5205
5206 LLVMContext &Ctx = F->getContext();
5207
5208 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5209 Type *RetTy = CI->getType();
5210 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5211 if (VT->getElementType()->isIntegerTy(16)) {
5212 VectorType *AsBF16 =
5213 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5214 Val = Builder.CreateBitCast(Val, AsBF16);
5215 }
5216 }
5217
5218 // The scope argument never really worked correctly. Use agent as the most
5219 // conservative option which should still always produce the instruction.
5220 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5221 AtomicRMWInst *RMW =
5222 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5223
5224 unsigned AddrSpace = PtrTy->getAddressSpace();
5225 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5226 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5227 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5228 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5229 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5230 }
5231
5232 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5233 MDBuilder MDB(F->getContext());
5234 MDNode *RangeNotPrivate =
5235 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5236 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5237 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5238 }
5239
5240 if (IsVolatile)
5241 RMW->setVolatile(true);
5242
5243 return Builder.CreateBitCast(RMW, RetTy);
5244}
5245
5246/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5247/// plain MDNode, as it's the verifier's job to check these are the correct
5248/// types later.
5249static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5250 if (Op < CI->arg_size()) {
5251 if (MetadataAsValue *MAV =
5252 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5253 Metadata *MD = MAV->getMetadata();
5254 return dyn_cast_if_present<MDNode>(MD);
5255 }
5256 }
5257 return nullptr;
5258}
5259
5260/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5261static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5262 if (Op < CI->arg_size())
5263 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5264 return MAV->getMetadata();
5265 return nullptr;
5266}
5267
5268/// Convert debug intrinsic calls to non-instruction debug records.
5269/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5270/// \p CI - The debug intrinsic call.
5271static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5272 DbgRecord *DR = nullptr;
5273 if (Name == "label") {
5274 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5275 } else if (Name == "assign") {
5276 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5277 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5278 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5279 unwrapMAVMetadataOp(CI, 4),
5280 /*The address is a Value ref, it will be stored as a Metadata */
5281 unwrapMAVOp(CI, 5));
5282 } else if (Name == "declare") {
5283 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5284 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5285 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5286 } else if (Name == "addr") {
5287 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5288 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5289 // Don't try to add something to the expression if it's not an expression.
5290 // Instead, allow the verifier to fail later.
5291 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5292 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5293 }
5294 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5295 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5296 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5297 } else if (Name == "value") {
5298 // An old version of dbg.value had an extra offset argument.
5299 unsigned VarOp = 1;
5300 unsigned ExprOp = 2;
5301 if (CI->arg_size() == 4) {
5302 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5303 // Nonzero offset dbg.values get dropped without a replacement.
5304 if (!Offset || !Offset->isNullValue())
5305 return;
5306 VarOp = 2;
5307 ExprOp = 3;
5308 }
5309 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5310 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5311 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5312 nullptr);
5313 }
5314 DR->setDebugLoc(CI->getDebugLoc());
5315 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5316 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5317}
5318
5319static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5320 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5321 if (!Offset)
5322 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5323 int64_t OffsetVal = Offset->getSExtValue();
5324 return Builder.CreateIntrinsic(OffsetVal >= 0
5325 ? Intrinsic::vector_splice_left
5326 : Intrinsic::vector_splice_right,
5327 CI->getType(),
5328 {CI->getArgOperand(0), CI->getArgOperand(1),
5329 Builder.getInt32(std::abs(OffsetVal))});
5330}
5331
5332static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5333 Function *F, IRBuilder<> &Builder) {
5334 if (Name.starts_with("to.fp16")) {
5335 Value *Cast =
5336 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5337 return Builder.CreateBitCast(Cast, CI->getType());
5338 }
5339
5340 if (Name.starts_with("from.fp16")) {
5341 Value *Cast =
5342 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5343 return Builder.CreateFPExt(Cast, CI->getType());
5344 }
5345
5346 return nullptr;
5347}
5348
5349static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5350 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5351 if (!MD || !isa<MDString>(MD))
5352 return ICmpInst::BAD_ICMP_PREDICATE;
5353 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5354 .Case("eq", ICmpInst::ICMP_EQ)
5355 .Case("ne", ICmpInst::ICMP_NE)
5356 .Case("ugt", ICmpInst::ICMP_UGT)
5357 .Case("uge", ICmpInst::ICMP_UGE)
5358 .Case("ult", ICmpInst::ICMP_ULT)
5359 .Case("ule", ICmpInst::ICMP_ULE)
5360 .Case("sgt", ICmpInst::ICMP_SGT)
5361 .Case("sge", ICmpInst::ICMP_SGE)
5362 .Case("slt", ICmpInst::ICMP_SLT)
5363 .Case("sle", ICmpInst::ICMP_SLE)
5364 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5365}
5366
5367static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5368 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5369 if (!MD || !isa<MDString>(MD))
5370 return FCmpInst::BAD_FCMP_PREDICATE;
5371 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5372 .Case("oeq", FCmpInst::FCMP_OEQ)
5373 .Case("ogt", FCmpInst::FCMP_OGT)
5374 .Case("oge", FCmpInst::FCMP_OGE)
5375 .Case("olt", FCmpInst::FCMP_OLT)
5376 .Case("ole", FCmpInst::FCMP_OLE)
5377 .Case("one", FCmpInst::FCMP_ONE)
5378 .Case("ord", FCmpInst::FCMP_ORD)
5379 .Case("uno", FCmpInst::FCMP_UNO)
5380 .Case("ueq", FCmpInst::FCMP_UEQ)
5381 .Case("ugt", FCmpInst::FCMP_UGT)
5382 .Case("uge", FCmpInst::FCMP_UGE)
5383 .Case("ult", FCmpInst::FCMP_ULT)
5384 .Case("ule", FCmpInst::FCMP_ULE)
5385 .Case("une", FCmpInst::FCMP_UNE)
5386 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5387}
5388
5389static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5390 IRBuilder<> &Builder) {
5391 Value *Rep;
5392 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5393 if (Opcode && Instruction::isUnaryOp(Opcode))
5394 Rep =
5395 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5396 else if (Opcode && Instruction::isBinaryOp(Opcode))
5397 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5398 CI->getArgOperand(0), CI->getArgOperand(1));
5399 else if (Opcode && Instruction::isCast(Opcode))
5400 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5401 CI->getType());
5402 else if (Opcode == Instruction::ICmp)
5403 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5404 CI->getArgOperand(0), CI->getArgOperand(1));
5405 else if (Opcode == Instruction::FCmp)
5406 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5407 CI->getArgOperand(0), CI->getArgOperand(1));
5408 else if (Opcode == Instruction::Select)
5409 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5410 CI->getArgOperand(2));
5411 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5412 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5413 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5414 } else
5415 llvm_unreachable("Unexpected vp intrinsic");
5416 Rep->takeName(CI);
5417 return Rep;
5418}
5419
5420static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5421 IRBuilder<> &Builder) {
5422 Intrinsic::ID IID = NewFn->getIntrinsicID();
5423
5424 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5425 if (Defaults.empty())
5426 return false;
5427
5428 unsigned OldArgCount = CI->arg_size();
5429 unsigned NewArgCount = NewFn->arg_size();
5430
5431 // If the caller already supplied all arguments (or more), nothing to do.
5432 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5433 if (OldArgCount >= NewArgCount)
5434 return false;
5435
5436 // Start with the existing arguments from the old call.
5437 SmallVector<Value *, 8> NewArgs(CI->args());
5438
5439 // Defaults are a contiguous trailing block, so checking the first missing
5440 // argument is enough.
5441 if (OldArgCount < FirstDefault)
5442 return false;
5443
5444 // Fill in each missing trailing argument from the table.
5445 FunctionType *NewFT = NewFn->getFunctionType();
5446 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5447 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5448 "missing argument outside the default range");
5449 Type *ParamTy = NewFT->getParamType(Idx);
5450
5451 // Only integer types are supported (i1, i8, i16, i32, i64).
5452 if (!ParamTy->isIntegerTy())
5453 return false;
5454 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5455 }
5456
5457 // Preserve operand bundles by creating the call with them.
5458 SmallVector<OperandBundleDef, 1> OpBundles;
5459 CI->getOperandBundlesAsDefs(OpBundles);
5460 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5461
5462 NewCall->takeName(CI);
5463 NewCall->setCallingConv(CI->getCallingConv());
5464 NewCall->copyMetadata(*CI);
5465 if (auto *OldCI = dyn_cast<CallInst>(CI))
5466 NewCall->setTailCallKind(OldCI->getTailCallKind());
5467
5468 CI->replaceAllUsesWith(NewCall);
5469 CI->eraseFromParent();
5470 return true;
5471}
5472
5473/// Upgrade a call to an old intrinsic. All argument and return casting must be
5474/// provided to seamlessly integrate with existing context.
5475void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5476 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5477 // checks the callee's function type matches. It's likely we need to handle
5478 // type changes here.
5479 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5480 if (!F)
5481 return;
5482
5483 LLVMContext &C = CI->getContext();
5484 IRBuilder<> Builder(C);
5485 if (isa<FPMathOperator>(CI))
5486 Builder.setFastMathFlags(CI->getFastMathFlags());
5487 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5488
5489 if (!NewFn) {
5490 // Get the Function's name.
5491 StringRef Name = F->getName();
5492 if (!Name.consume_front("llvm."))
5493 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5494
5495 bool IsX86 = Name.consume_front("x86.");
5496 bool IsNVVM = Name.consume_front("nvvm.");
5497 bool IsAArch64 = Name.consume_front("aarch64.");
5498 bool IsARM = Name.consume_front("arm.");
5499 bool IsAMDGCN = Name.consume_front("amdgcn.");
5500 bool IsDbg = Name.consume_front("dbg.");
5501 bool IsOldSplice =
5502 (Name.consume_front("experimental.vector.splice") ||
5503 Name.consume_front("vector.splice")) &&
5504 !(Name.starts_with(".left") || Name.starts_with(".right"));
5505 Value *Rep = nullptr;
5506
5507 if (!IsX86 && Name == "stackprotectorcheck") {
5508 Rep = nullptr;
5509 } else if (IsNVVM) {
5510 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5511 } else if (IsX86) {
5512 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5513 } else if (IsAArch64) {
5514 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5515 } else if (IsARM) {
5516 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5517 } else if (IsAMDGCN) {
5518 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5519 } else if (IsDbg) {
5520 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5521 } else if (IsOldSplice) {
5522 Rep = upgradeVectorSplice(CI, Builder);
5523 } else if (Name.consume_front("convert.")) {
5524 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5525 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5526 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5527 Rep = nullptr;
5528 } else if (shouldUpgradeVPIntrinsic(Name)) {
5529 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5530 } else {
5531 llvm_unreachable("Unknown function for CallBase upgrade.");
5532 }
5533
5534 if (Rep)
5535 CI->replaceAllUsesWith(Rep);
5536 CI->eraseFromParent();
5537 return;
5538 }
5539
5540 const auto &DefaultCase = [&]() -> void {
5541 if (F == NewFn)
5542 return;
5543
5544 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5545 // Handle generic mangling change.
5546 assert(
5547 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5548 "Unknown function for CallBase upgrade and isn't just a name change");
5549 CI->setCalledFunction(NewFn);
5550 return;
5551 }
5552
5553 // This must be an upgrade from a named to a literal struct.
5554 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5555 assert(OldST != NewFn->getReturnType() &&
5556 "Return type must have changed");
5557 assert(OldST->getNumElements() ==
5558 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5559 "Must have same number of elements");
5560
5561 SmallVector<Value *> Args(CI->args());
5562 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5563 NewCI->setAttributes(CI->getAttributes());
5564 Value *Res = PoisonValue::get(OldST);
5565 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5566 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5567 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5568 }
5569 CI->replaceAllUsesWith(Res);
5570 CI->eraseFromParent();
5571 return;
5572 }
5573
5574 // We're probably about to produce something invalid. Let the verifier catch
5575 // it instead of dying here.
5576 CI->setCalledOperand(
5577 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5578 return;
5579 };
5580 CallInst *NewCall = nullptr;
5581 switch (NewFn->getIntrinsicID()) {
5582 default: {
5583 // Last resort: try the data-driven default-arg upgrade.
5584 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5585 // in its .td definition, without needing a dedicated case.
5586 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5587 return;
5588 DefaultCase();
5589 return;
5590 }
5591 case Intrinsic::arm_neon_vst1:
5592 case Intrinsic::arm_neon_vst2:
5593 case Intrinsic::arm_neon_vst3:
5594 case Intrinsic::arm_neon_vst4:
5595 case Intrinsic::arm_neon_vst2lane:
5596 case Intrinsic::arm_neon_vst3lane:
5597 case Intrinsic::arm_neon_vst4lane: {
5598 SmallVector<Value *, 4> Args(CI->args());
5599 NewCall = Builder.CreateCall(NewFn, Args);
5600 break;
5601 }
5602 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5603 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5604 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5605 LLVMContext &Ctx = F->getParent()->getContext();
5606 SmallVector<Value *, 4> Args(CI->args());
5607 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5608 cast<ConstantInt>(Args[3])->getZExtValue());
5609 NewCall = Builder.CreateCall(NewFn, Args);
5610 break;
5611 }
5612 case Intrinsic::aarch64_sve_ld3_sret:
5613 case Intrinsic::aarch64_sve_ld4_sret:
5614 case Intrinsic::aarch64_sve_ld2_sret: {
5615 // Is this a trivial remangle of the name to support ptr address spaces?
5616 if (isa<StructType>(F->getReturnType())) {
5617 DefaultCase();
5618 return;
5619 }
5620
5621 StringRef Name = F->getName();
5622 Name = Name.substr(5);
5623 unsigned N = StringSwitch<unsigned>(Name)
5624 .StartsWith("aarch64.sve.ld2", 2)
5625 .StartsWith("aarch64.sve.ld3", 3)
5626 .StartsWith("aarch64.sve.ld4", 4)
5627 .Default(0);
5628 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5629 unsigned MinElts = RetTy->getMinNumElements() / N;
5630 SmallVector<Value *, 2> Args(CI->args());
5631 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5632 Value *Ret = llvm::PoisonValue::get(RetTy);
5633 for (unsigned I = 0; I < N; I++) {
5634 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5635 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5636 }
5637 NewCall = dyn_cast<CallInst>(Ret);
5638 break;
5639 }
5640
5641 case Intrinsic::coro_end_async:
5642 case Intrinsic::coro_end: {
5643 SmallVector<Value *, 3> Args(CI->args());
5644 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5645 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5646 NewCall = Builder.CreateCall(NewFn, Args);
5647
5648 if (!CI->getType()->isVoidTy()) {
5649 if (!CI->use_empty()) {
5650 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5651 CI->getModule(), Intrinsic::coro_is_in_ramp);
5652 Value *InRamp = Builder.CreateCall(IsInRamp);
5653 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5654 }
5655 CI->eraseFromParent();
5656 return;
5657 }
5658
5659 break;
5660 }
5661
5662 case Intrinsic::vector_extract: {
5663 StringRef Name = F->getName();
5664 Name = Name.substr(5); // Strip llvm
5665 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5666 DefaultCase();
5667 return;
5668 }
5669 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5670 unsigned MinElts = RetTy->getMinNumElements();
5671 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5672 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5673 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5674 break;
5675 }
5676
5677 case Intrinsic::vector_insert: {
5678 StringRef Name = F->getName();
5679 Name = Name.substr(5);
5680 if (!Name.starts_with("aarch64.sve.tuple")) {
5681 DefaultCase();
5682 return;
5683 }
5684 if (Name.starts_with("aarch64.sve.tuple.set")) {
5685 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5686 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5687 Value *NewIdx =
5688 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5689 NewCall = Builder.CreateCall(
5690 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5691 break;
5692 }
5693 if (Name.starts_with("aarch64.sve.tuple.create")) {
5694 unsigned N = StringSwitch<unsigned>(Name)
5695 .StartsWith("aarch64.sve.tuple.create2", 2)
5696 .StartsWith("aarch64.sve.tuple.create3", 3)
5697 .StartsWith("aarch64.sve.tuple.create4", 4)
5698 .Default(0);
5699 assert(N > 1 && "Create is expected to be between 2-4");
5700 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5701 Value *Ret = llvm::PoisonValue::get(RetTy);
5702 unsigned MinElts = RetTy->getMinNumElements() / N;
5703 for (unsigned I = 0; I < N; I++) {
5704 Value *V = CI->getArgOperand(I);
5705 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5706 }
5707 NewCall = dyn_cast<CallInst>(Ret);
5708 }
5709 break;
5710 }
5711
5712 case Intrinsic::arm_neon_bfdot:
5713 case Intrinsic::arm_neon_bfmmla:
5714 case Intrinsic::arm_neon_bfmlalb:
5715 case Intrinsic::arm_neon_bfmlalt:
5716 case Intrinsic::aarch64_neon_bfdot:
5717 case Intrinsic::aarch64_neon_bfmmla:
5718 case Intrinsic::aarch64_neon_bfmlalb:
5719 case Intrinsic::aarch64_neon_bfmlalt: {
5720 SmallVector<Value *, 3> Args;
5721 assert(CI->arg_size() == 3 &&
5722 "Mismatch between function args and call args");
5723 size_t OperandWidth =
5724 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5725 assert((OperandWidth == 64 || OperandWidth == 128) &&
5726 "Unexpected operand width");
5727 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5728 auto Iter = CI->args().begin();
5729 Args.push_back(*Iter++);
5730 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5731 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5732 NewCall = Builder.CreateCall(NewFn, Args);
5733 break;
5734 }
5735
5736 case Intrinsic::bitreverse:
5737 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5738 break;
5739
5740 case Intrinsic::ctlz:
5741 case Intrinsic::cttz: {
5742 if (CI->arg_size() != 1) {
5743 DefaultCase();
5744 return;
5745 }
5746
5747 NewCall =
5748 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5749 break;
5750 }
5751
5752 case Intrinsic::objectsize: {
5753 Value *NullIsUnknownSize =
5754 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5755 Value *Dynamic =
5756 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5757 NewCall = Builder.CreateCall(
5758 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5759 break;
5760 }
5761
5762 case Intrinsic::ctpop:
5763 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5764 break;
5765 case Intrinsic::dbg_value: {
5766 StringRef Name = F->getName();
5767 Name = Name.substr(5); // Strip llvm.
5768 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5769 if (Name.starts_with("dbg.addr")) {
5770 DIExpression *Expr = cast<DIExpression>(
5771 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5772 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5773 NewCall =
5774 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5775 MetadataAsValue::get(C, Expr)});
5776 break;
5777 }
5778
5779 // Upgrade from the old version that had an extra offset argument.
5780 assert(CI->arg_size() == 4);
5781 // Drop nonzero offsets instead of attempting to upgrade them.
5782 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5783 if (Offset->isNullValue()) {
5784 NewCall = Builder.CreateCall(
5785 NewFn,
5786 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5787 break;
5788 }
5789 CI->eraseFromParent();
5790 return;
5791 }
5792
5793 case Intrinsic::ptr_annotation:
5794 // Upgrade from versions that lacked the annotation attribute argument.
5795 if (CI->arg_size() != 4) {
5796 DefaultCase();
5797 return;
5798 }
5799
5800 // Create a new call with an added null annotation attribute argument.
5801 NewCall = Builder.CreateCall(
5802 NewFn,
5803 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5804 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5805 NewCall->takeName(CI);
5806 CI->replaceAllUsesWith(NewCall);
5807 CI->eraseFromParent();
5808 return;
5809
5810 case Intrinsic::var_annotation:
5811 // Upgrade from versions that lacked the annotation attribute argument.
5812 if (CI->arg_size() != 4) {
5813 DefaultCase();
5814 return;
5815 }
5816 // Create a new call with an added null annotation attribute argument.
5817 NewCall = Builder.CreateCall(
5818 NewFn,
5819 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5820 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5821 NewCall->takeName(CI);
5822 CI->replaceAllUsesWith(NewCall);
5823 CI->eraseFromParent();
5824 return;
5825
5826 case Intrinsic::riscv_aes32dsi:
5827 case Intrinsic::riscv_aes32dsmi:
5828 case Intrinsic::riscv_aes32esi:
5829 case Intrinsic::riscv_aes32esmi:
5830 case Intrinsic::riscv_sm4ks:
5831 case Intrinsic::riscv_sm4ed: {
5832 // The last argument to these intrinsics used to be i8 and changed to i32.
5833 // The type overload for sm4ks and sm4ed was removed.
5834 Value *Arg2 = CI->getArgOperand(2);
5835 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5836 return;
5837
5838 Value *Arg0 = CI->getArgOperand(0);
5839 Value *Arg1 = CI->getArgOperand(1);
5840 if (CI->getType()->isIntegerTy(64)) {
5841 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5842 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5843 }
5844
5845 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5846 cast<ConstantInt>(Arg2)->getZExtValue());
5847
5848 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5849 Value *Res = NewCall;
5850 if (Res->getType() != CI->getType())
5851 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5852 NewCall->takeName(CI);
5853 CI->replaceAllUsesWith(Res);
5854 CI->eraseFromParent();
5855 return;
5856 }
5857 case Intrinsic::nvvm_mapa_shared_cluster: {
5858 // Create a new call with the correct address space.
5859 NewCall =
5860 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5861 Value *Res = NewCall;
5862 Res = Builder.CreateAddrSpaceCast(
5863 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5864 NewCall->takeName(CI);
5865 CI->replaceAllUsesWith(Res);
5866 CI->eraseFromParent();
5867 return;
5868 }
5869 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5870 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5871 // Create a new call with the correct address space.
5872 SmallVector<Value *, 4> Args(CI->args());
5873 Args[0] = Builder.CreateAddrSpaceCast(
5874 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5875
5876 NewCall = Builder.CreateCall(NewFn, Args);
5877 NewCall->takeName(CI);
5878 CI->replaceAllUsesWith(NewCall);
5879 CI->eraseFromParent();
5880 return;
5881 }
5882 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5883 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5884 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5885 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5886 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5887 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5888 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5889 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5890 SmallVector<Value *, 16> Args(CI->args());
5891
5892 // Create AddrSpaceCast to shared_cluster if needed.
5893 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5894 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5895 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5896 Args[0] = Builder.CreateAddrSpaceCast(
5897 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5898
5899 // Attach the flag argument for cta_group, with a
5900 // default value of 0. This handles case (2) in
5901 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5902 size_t NumArgs = CI->arg_size();
5903 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5904 if (!FlagArg->getType()->isIntegerTy(1))
5905 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5906
5907 NewCall = Builder.CreateCall(NewFn, Args);
5908 NewCall->takeName(CI);
5909 CI->replaceAllUsesWith(NewCall);
5910 CI->eraseFromParent();
5911 return;
5912 }
5913 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
5914 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
5915 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
5916 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
5917 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
5918 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
5919 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
5920 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
5921 StringRef Name = F->getName();
5922 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
5923 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
5924
5925 SmallVector<Value *, 16> Args(CI->args());
5926 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
5927 NewCall = Builder.CreateCall(NewFn, Args);
5928 break;
5929 }
5930 case Intrinsic::nvvm_tcgen05_mma_shared:
5931 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5932 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5933 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
5934 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
5935 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
5936 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
5937 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
5938 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
5939 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
5940 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5941 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5942 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
5943 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5944 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5945 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
5946 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
5947 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
5948 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
5949 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
5950 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
5951 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
5952 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5953 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5954 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
5955 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
5956 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5957 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5958 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5959 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
5960 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
5961 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
5962 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
5963 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
5964 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
5965 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
5966 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
5967 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
5968 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
5969 case Intrinsic::
5970 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
5971 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
5972 case Intrinsic::
5973 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
5974 case Intrinsic::nvvm_tcgen05_mma_tensor:
5975 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
5976 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
5977 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
5978 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
5979 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
5980 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
5981 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
5982 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
5983 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
5984 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
5985 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
5986 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
5987 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
5988 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
5989 case Intrinsic::
5990 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
5991 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
5992 case Intrinsic::
5993 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
5994 SmallVector<Value *, 12> Args(CI->args());
5995 Args.push_back(Builder.getInt32(0)); // collector_usage_b = discard(0)
5996 NewCall = Builder.CreateCall(NewFn, Args);
5997 break;
5998 }
5999 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6000 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6001 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6002 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6003 NewCall =
6004 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6005 Builder.getFalse()});
6006 break;
6007 case Intrinsic::riscv_sha256sig0:
6008 case Intrinsic::riscv_sha256sig1:
6009 case Intrinsic::riscv_sha256sum0:
6010 case Intrinsic::riscv_sha256sum1:
6011 case Intrinsic::riscv_sm3p0:
6012 case Intrinsic::riscv_sm3p1: {
6013 // The last argument to these intrinsics used to be i8 and changed to i32.
6014 // The type overload for sm4ks and sm4ed was removed.
6015 if (!CI->getType()->isIntegerTy(64))
6016 return;
6017
6018 Value *Arg =
6019 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6020
6021 NewCall = Builder.CreateCall(NewFn, Arg);
6022 Value *Res =
6023 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6024 NewCall->takeName(CI);
6025 CI->replaceAllUsesWith(Res);
6026 CI->eraseFromParent();
6027 return;
6028 }
6029
6030 case Intrinsic::x86_xop_vfrcz_ss:
6031 case Intrinsic::x86_xop_vfrcz_sd:
6032 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6033 break;
6034
6035 case Intrinsic::x86_xop_vpermil2pd:
6036 case Intrinsic::x86_xop_vpermil2ps:
6037 case Intrinsic::x86_xop_vpermil2pd_256:
6038 case Intrinsic::x86_xop_vpermil2ps_256: {
6039 SmallVector<Value *, 4> Args(CI->args());
6040 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6041 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6042 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6043 NewCall = Builder.CreateCall(NewFn, Args);
6044 break;
6045 }
6046
6047 case Intrinsic::x86_sse41_ptestc:
6048 case Intrinsic::x86_sse41_ptestz:
6049 case Intrinsic::x86_sse41_ptestnzc: {
6050 // The arguments for these intrinsics used to be v4f32, and changed
6051 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6052 // So, the only thing required is a bitcast for both arguments.
6053 // First, check the arguments have the old type.
6054 Value *Arg0 = CI->getArgOperand(0);
6055 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6056 return;
6057
6058 // Old intrinsic, add bitcasts
6059 Value *Arg1 = CI->getArgOperand(1);
6060
6061 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6062
6063 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6064 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6065
6066 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6067 break;
6068 }
6069
6070 case Intrinsic::x86_rdtscp: {
6071 // This used to take 1 arguments. If we have no arguments, it is already
6072 // upgraded.
6073 if (CI->getNumOperands() == 0)
6074 return;
6075
6076 NewCall = Builder.CreateCall(NewFn);
6077 // Extract the second result and store it.
6078 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6079 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6080 // Replace the original call result with the first result of the new call.
6081 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6082
6083 NewCall->takeName(CI);
6084 CI->replaceAllUsesWith(TSC);
6085 CI->eraseFromParent();
6086 return;
6087 }
6088
6089 case Intrinsic::x86_sse41_insertps:
6090 case Intrinsic::x86_sse41_dppd:
6091 case Intrinsic::x86_sse41_dpps:
6092 case Intrinsic::x86_sse41_mpsadbw:
6093 case Intrinsic::x86_avx_dp_ps_256:
6094 case Intrinsic::x86_avx2_mpsadbw: {
6095 // Need to truncate the last argument from i32 to i8 -- this argument models
6096 // an inherently 8-bit immediate operand to these x86 instructions.
6097 SmallVector<Value *, 4> Args(CI->args());
6098
6099 // Replace the last argument with a trunc.
6100 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6101 NewCall = Builder.CreateCall(NewFn, Args);
6102 break;
6103 }
6104
6105 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6106 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6107 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6108 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6109 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6110 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6111 SmallVector<Value *, 4> Args(CI->args());
6112 unsigned NumElts =
6113 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6114 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6115
6116 NewCall = Builder.CreateCall(NewFn, Args);
6117 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6118
6119 NewCall->takeName(CI);
6120 CI->replaceAllUsesWith(Res);
6121 CI->eraseFromParent();
6122 return;
6123 }
6124
6125 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6126 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6127 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6128 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6129 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6130 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6131 SmallVector<Value *, 4> Args(CI->args());
6132 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6133 if (NewFn->getIntrinsicID() ==
6134 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6135 Args[1] = Builder.CreateBitCast(
6136 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6137
6138 NewCall = Builder.CreateCall(NewFn, Args);
6139 Value *Res = Builder.CreateBitCast(
6140 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6141
6142 NewCall->takeName(CI);
6143 CI->replaceAllUsesWith(Res);
6144 CI->eraseFromParent();
6145 return;
6146 }
6147 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6148 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6149 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6150 SmallVector<Value *, 4> Args(CI->args());
6151 unsigned NumElts =
6152 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6153 Args[1] = Builder.CreateBitCast(
6154 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6155 Args[2] = Builder.CreateBitCast(
6156 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6157
6158 NewCall = Builder.CreateCall(NewFn, Args);
6159 break;
6160 }
6161
6162 case Intrinsic::thread_pointer: {
6163 NewCall = Builder.CreateCall(NewFn, {});
6164 break;
6165 }
6166
6167 case Intrinsic::memcpy:
6168 case Intrinsic::memmove:
6169 case Intrinsic::memset: {
6170 // We have to make sure that the call signature is what we're expecting.
6171 // We only want to change the old signatures by removing the alignment arg:
6172 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6173 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6174 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6175 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6176 // Note: i8*'s in the above can be any pointer type
6177 if (CI->arg_size() != 5) {
6178 DefaultCase();
6179 return;
6180 }
6181 // Remove alignment argument (3), and add alignment attributes to the
6182 // dest/src pointers.
6183 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6184 CI->getArgOperand(2), CI->getArgOperand(4)};
6185 NewCall = Builder.CreateCall(NewFn, Args);
6186 AttributeList OldAttrs = CI->getAttributes();
6187 AttributeList NewAttrs = AttributeList::get(
6188 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6189 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6190 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6191 NewCall->setAttributes(NewAttrs);
6192 auto *MemCI = cast<MemIntrinsic>(NewCall);
6193 // All mem intrinsics support dest alignment.
6194 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6195 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6196 // Memcpy/Memmove also support source alignment.
6197 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6198 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6199 break;
6200 }
6201
6202 case Intrinsic::masked_load:
6203 case Intrinsic::masked_gather:
6204 case Intrinsic::masked_store:
6205 case Intrinsic::masked_scatter: {
6206 if (CI->arg_size() != 4) {
6207 DefaultCase();
6208 return;
6209 }
6210
6211 auto GetMaybeAlign = [](Value *Op) {
6212 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6213 uint64_t Val = CI->getZExtValue();
6214 if (Val == 0)
6215 return MaybeAlign();
6216 if (isPowerOf2_64(Val))
6217 return MaybeAlign(Val);
6218 }
6219 reportFatalUsageError("Invalid alignment argument");
6220 };
6221 auto GetAlign = [&](Value *Op) {
6222 MaybeAlign Align = GetMaybeAlign(Op);
6223 if (Align)
6224 return *Align;
6225 reportFatalUsageError("Invalid zero alignment argument");
6226 };
6227
6228 const DataLayout &DL = CI->getDataLayout();
6229 switch (NewFn->getIntrinsicID()) {
6230 case Intrinsic::masked_load:
6231 NewCall = Builder.CreateMaskedLoad(
6232 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6233 CI->getArgOperand(2), CI->getArgOperand(3));
6234 break;
6235 case Intrinsic::masked_gather:
6236 NewCall = Builder.CreateMaskedGather(
6237 CI->getType(), CI->getArgOperand(0),
6238 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6239 CI->getType()->getScalarType()),
6240 CI->getArgOperand(2), CI->getArgOperand(3));
6241 break;
6242 case Intrinsic::masked_store:
6243 NewCall = Builder.CreateMaskedStore(
6244 CI->getArgOperand(0), CI->getArgOperand(1),
6245 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6246 break;
6247 case Intrinsic::masked_scatter:
6248 NewCall = Builder.CreateMaskedScatter(
6249 CI->getArgOperand(0), CI->getArgOperand(1),
6250 DL.getValueOrABITypeAlignment(
6251 GetMaybeAlign(CI->getArgOperand(2)),
6252 CI->getArgOperand(0)->getType()->getScalarType()),
6253 CI->getArgOperand(3));
6254 break;
6255 default:
6256 llvm_unreachable("Unexpected intrinsic ID");
6257 }
6258 // Previous metadata is still valid.
6259 NewCall->copyMetadata(*CI);
6260 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6261 break;
6262 }
6263
6264 case Intrinsic::lifetime_start:
6265 case Intrinsic::lifetime_end: {
6266 if (CI->arg_size() != 2) {
6267 DefaultCase();
6268 return;
6269 }
6270
6271 Value *Ptr = CI->getArgOperand(1);
6272 // Try to strip pointer casts, such that the lifetime works on an alloca.
6273 Ptr = Ptr->stripPointerCasts();
6274 if (isa<AllocaInst>(Ptr)) {
6275 // Don't use NewFn, as we might have looked through an addrspacecast.
6276 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6277 NewCall = Builder.CreateLifetimeStart(Ptr);
6278 else
6279 NewCall = Builder.CreateLifetimeEnd(Ptr);
6280 break;
6281 }
6282
6283 // Otherwise remove the lifetime marker.
6284 CI->eraseFromParent();
6285 return;
6286 }
6287
6288 case Intrinsic::x86_avx512_vpdpbusd_128:
6289 case Intrinsic::x86_avx512_vpdpbusd_256:
6290 case Intrinsic::x86_avx512_vpdpbusd_512:
6291 case Intrinsic::x86_avx512_vpdpbusds_128:
6292 case Intrinsic::x86_avx512_vpdpbusds_256:
6293 case Intrinsic::x86_avx512_vpdpbusds_512:
6294 case Intrinsic::x86_avx2_vpdpbssd_128:
6295 case Intrinsic::x86_avx2_vpdpbssd_256:
6296 case Intrinsic::x86_avx10_vpdpbssd_512:
6297 case Intrinsic::x86_avx2_vpdpbssds_128:
6298 case Intrinsic::x86_avx2_vpdpbssds_256:
6299 case Intrinsic::x86_avx10_vpdpbssds_512:
6300 case Intrinsic::x86_avx2_vpdpbsud_128:
6301 case Intrinsic::x86_avx2_vpdpbsud_256:
6302 case Intrinsic::x86_avx10_vpdpbsud_512:
6303 case Intrinsic::x86_avx2_vpdpbsuds_128:
6304 case Intrinsic::x86_avx2_vpdpbsuds_256:
6305 case Intrinsic::x86_avx10_vpdpbsuds_512:
6306 case Intrinsic::x86_avx2_vpdpbuud_128:
6307 case Intrinsic::x86_avx2_vpdpbuud_256:
6308 case Intrinsic::x86_avx10_vpdpbuud_512:
6309 case Intrinsic::x86_avx2_vpdpbuuds_128:
6310 case Intrinsic::x86_avx2_vpdpbuuds_256:
6311 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6312 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6313 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6314 CI->getArgOperand(2)};
6315 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6316 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6317 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6318
6319 NewCall = Builder.CreateCall(NewFn, Args);
6320 break;
6321 }
6322 case Intrinsic::x86_avx512_vpdpwssd_128:
6323 case Intrinsic::x86_avx512_vpdpwssd_256:
6324 case Intrinsic::x86_avx512_vpdpwssd_512:
6325 case Intrinsic::x86_avx512_vpdpwssds_128:
6326 case Intrinsic::x86_avx512_vpdpwssds_256:
6327 case Intrinsic::x86_avx512_vpdpwssds_512:
6328 case Intrinsic::x86_avx2_vpdpwsud_128:
6329 case Intrinsic::x86_avx2_vpdpwsud_256:
6330 case Intrinsic::x86_avx10_vpdpwsud_512:
6331 case Intrinsic::x86_avx2_vpdpwsuds_128:
6332 case Intrinsic::x86_avx2_vpdpwsuds_256:
6333 case Intrinsic::x86_avx10_vpdpwsuds_512:
6334 case Intrinsic::x86_avx2_vpdpwusd_128:
6335 case Intrinsic::x86_avx2_vpdpwusd_256:
6336 case Intrinsic::x86_avx10_vpdpwusd_512:
6337 case Intrinsic::x86_avx2_vpdpwusds_128:
6338 case Intrinsic::x86_avx2_vpdpwusds_256:
6339 case Intrinsic::x86_avx10_vpdpwusds_512:
6340 case Intrinsic::x86_avx2_vpdpwuud_128:
6341 case Intrinsic::x86_avx2_vpdpwuud_256:
6342 case Intrinsic::x86_avx10_vpdpwuud_512:
6343 case Intrinsic::x86_avx2_vpdpwuuds_128:
6344 case Intrinsic::x86_avx2_vpdpwuuds_256:
6345 case Intrinsic::x86_avx10_vpdpwuuds_512:
6346 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6347 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6348 CI->getArgOperand(2)};
6349 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6350 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6351 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6352
6353 NewCall = Builder.CreateCall(NewFn, Args);
6354 break;
6355 }
6356 assert(NewCall && "Should have either set this variable or returned through "
6357 "the default case");
6358 NewCall->takeName(CI);
6359 CI->replaceAllUsesWith(NewCall);
6360 CI->eraseFromParent();
6361}
6362
6363void llvm::UpgradeCallsToIntrinsic(Function *F) {
6364 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6365
6366 // Check if this function should be upgraded and get the replacement function
6367 // if there is one.
6368 Function *NewFn;
6369 if (UpgradeIntrinsicFunction(F, NewFn)) {
6370 // Replace all users of the old function with the new function or new
6371 // instructions. This is not a range loop because the call is deleted.
6372 for (User *U : make_early_inc_range(F->users()))
6373 if (CallBase *CB = dyn_cast<CallBase>(U))
6374 UpgradeIntrinsicCall(CB, NewFn);
6375
6376 // Remove old function, no longer used, from the module.
6377 if (F != NewFn)
6378 F->eraseFromParent();
6379 }
6380}
6381
6382MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6383 const unsigned NumOperands = MD.getNumOperands();
6384 if (NumOperands == 0)
6385 return &MD; // Invalid, punt to a verifier error.
6386
6387 // Check if the tag uses struct-path aware TBAA format.
6388 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6389 return &MD;
6390
6391 auto &Context = MD.getContext();
6392 if (NumOperands == 3) {
6393 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6394 MDNode *ScalarType = MDNode::get(Context, Elts);
6395 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6396 Metadata *Elts2[] = {ScalarType, ScalarType,
6397 ConstantAsMetadata::get(
6398 Constant::getNullValue(Type::getInt64Ty(Context))),
6399 MD.getOperand(2)};
6400 return MDNode::get(Context, Elts2);
6401 }
6402 // Create a MDNode <MD, MD, offset 0>
6403 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6404 Type::getInt64Ty(Context)))};
6405 return MDNode::get(Context, Elts);
6406}
6407
6408Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6409 Instruction *&Temp) {
6410 if (Opc != Instruction::BitCast)
6411 return nullptr;
6412
6413 Temp = nullptr;
6414 Type *SrcTy = V->getType();
6415 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6416 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6417 LLVMContext &Context = V->getContext();
6418
6419 // We have no information about target data layout, so we assume that
6420 // the maximum pointer size is 64bit.
6421 Type *MidTy = Type::getInt64Ty(Context);
6422 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6423
6424 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6425 }
6426
6427 return nullptr;
6428}
6429
6430Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6431 if (Opc != Instruction::BitCast)
6432 return nullptr;
6433
6434 Type *SrcTy = C->getType();
6435 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6436 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6437 LLVMContext &Context = C->getContext();
6438
6439 // We have no information about target data layout, so we assume that
6440 // the maximum pointer size is 64bit.
6441 Type *MidTy = Type::getInt64Ty(Context);
6442
6443 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6444 DestTy);
6445 }
6446
6447 return nullptr;
6448}
6449
6450static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6451 if (Flag.getNumOperands() < 3)
6452 return std::nullopt;
6453 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6454 return Name->getString();
6455 return std::nullopt;
6456}
6457
6458/// Check the debug info version number, if it is out-dated, drop the debug
6459/// info. Return true if module is modified.
6460bool llvm::UpgradeDebugInfo(Module &M) {
6461 if (DisableAutoUpgradeDebugInfo)
6462 return false;
6463
6464 llvm::TimeTraceScope timeScope("Upgrade debug info");
6465 // We need to get metadata before the module is verified (i.e., getModuleFlag
6466 // makes assumptions that we haven't verified yet). Carefully extract the flag
6467 // from the metadata.
6468 unsigned Version = 0;
6469 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6470 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6471 if (auto Name = getModuleFlagNameSafely(*Flag))
6472 return *Name == "Debug Info Version";
6473 return false;
6474 });
6475 if (OpIt != ModFlags->op_end()) {
6476 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6477 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6478 Version = CI->getZExtValue();
6479 }
6480 }
6481
6482 if (Version == DEBUG_METADATA_VERSION) {
6483 bool BrokenDebugInfo = false;
6484 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6485 report_fatal_error("Broken module found, compilation aborted!");
6486 if (!BrokenDebugInfo)
6487 // Everything is ok.
6488 return false;
6489 else {
6490 // Diagnose malformed debug info.
6491 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6492 M.getContext().diagnose(Diag);
6493 }
6494 }
6495 bool Modified = StripDebugInfo(M);
6496 if (Modified && Version != DEBUG_METADATA_VERSION) {
6497 // Diagnose a version mismatch.
6498 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6499 M.getContext().diagnose(DiagVersion);
6500 }
6501 return Modified;
6502}
6503
6504static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6505 GlobalValue *GV, const Metadata *V) {
6506 Function *F = cast<Function>(GV);
6507
6508 constexpr StringLiteral DefaultValue = "1";
6509 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6510 unsigned Length = 0;
6511
6512 if (F->hasFnAttribute(Attr)) {
6513 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6514 // parse these elements placing them into Vect3
6515 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6516 for (; Length < 3 && !S.empty(); Length++) {
6517 auto [Part, Rest] = S.split(',');
6518 Vect3[Length] = Part.trim();
6519 S = Rest;
6520 }
6521 }
6522
6523 const unsigned Dim = DimC - 'x';
6524 assert(Dim < 3 && "Unexpected dim char");
6525
6526 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6527
6528 // local variable required for StringRef in Vect3 to point to.
6529 const std::string VStr = llvm::utostr(VInt);
6530 Vect3[Dim] = VStr;
6531 Length = std::max(Length, Dim + 1);
6532
6533 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6534 F->addFnAttr(Attr, NewAttr);
6535}
6536
6537static inline bool isXYZ(StringRef S) {
6538 return S == "x" || S == "y" || S == "z";
6539}
6540
6541bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6542 const Metadata *V) {
6543 if (K == "kernel") {
6544 if (!mdconst::extract<ConstantInt>(V)->isZero())
6545 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6546 return true;
6547 }
6548 if (K == "align") {
6549 // V is a bitfeild specifying two 16-bit values. The alignment value is
6550 // specfied in low 16-bits, The index is specified in the high bits. For the
6551 // index, 0 indicates the return value while higher values correspond to
6552 // each parameter (idx = param + 1).
6553 const uint64_t AlignIdxValuePair =
6554 mdconst::extract<ConstantInt>(V)->getZExtValue();
6555 const unsigned Idx = (AlignIdxValuePair >> 16);
6556 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6557 cast<Function>(GV)->addAttributeAtIndex(
6558 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6559 return true;
6560 }
6561 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6562 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6563 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6564 return true;
6565 }
6566 if (K == "minctasm") {
6567 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6568 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6569 return true;
6570 }
6571 if (K == "maxnreg") {
6572 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6573 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6574 return true;
6575 }
6576 if (K.consume_front("maxntid") && isXYZ(K)) {
6577 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6578 return true;
6579 }
6580 if (K.consume_front("reqntid") && isXYZ(K)) {
6581 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6582 return true;
6583 }
6584 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6585 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6586 return true;
6587 }
6588 if (K == "grid_constant") {
6589 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6590 for (const auto &Op : cast<MDNode>(V)->operands()) {
6591 // For some reason, the index is 1-based in the metadata. Good thing we're
6592 // able to auto-upgrade it!
6593 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6594 cast<Function>(GV)->addParamAttr(Index, Attr);
6595 }
6596 return true;
6597 }
6598
6599 return false;
6600}
6601
6602void llvm::UpgradeNVVMAnnotations(Module &M) {
6603 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6604 if (!NamedMD)
6605 return;
6606
6607 SmallVector<MDNode *, 8> NewNodes;
6608 SmallPtrSet<const MDNode *, 8> SeenNodes;
6609 for (MDNode *MD : NamedMD->operands()) {
6610 if (!SeenNodes.insert(MD).second)
6611 continue;
6612
6613 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6614 if (!GV)
6615 continue;
6616
6617 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6618
6619 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6620 // Each nvvm.annotations metadata entry will be of the following form:
6621 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6622 // start index = 1, to skip the global variable key
6623 // increment = 2, to skip the value for each property-value pairs
6624 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6625 MDString *K = cast<MDString>(MD->getOperand(j));
6626 const MDOperand &V = MD->getOperand(j + 1);
6627 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6628 if (!Upgraded)
6629 NewOperands.append({K, V});
6630 }
6631
6632 if (NewOperands.size() > 1)
6633 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6634 }
6635
6636 NamedMD->clearOperands();
6637 for (MDNode *N : NewNodes)
6638 NamedMD->addOperand(N);
6639}
6640
6641/// This checks for objc retain release marker which should be upgraded. It
6642/// returns true if module is modified.
6643static bool upgradeRetainReleaseMarker(Module &M) {
6644 bool Changed = false;
6645 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6646 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6647 if (ModRetainReleaseMarker) {
6648 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6649 if (Op) {
6650 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6651 if (ID) {
6652 SmallVector<StringRef, 4> ValueComp;
6653 ID->getString().split(ValueComp, "#");
6654 if (ValueComp.size() == 2) {
6655 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6656 ID = MDString::get(M.getContext(), NewValue);
6657 }
6658 M.addModuleFlag(Module::Error, MarkerKey, ID);
6659 M.eraseNamedMetadata(ModRetainReleaseMarker);
6660 Changed = true;
6661 }
6662 }
6663 }
6664 return Changed;
6665}
6666
6667void llvm::UpgradeARCRuntime(Module &M) {
6668 // This lambda converts normal function calls to ARC runtime functions to
6669 // intrinsic calls.
6670 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6671 llvm::Intrinsic::ID IntrinsicFunc) {
6672 Function *Fn = M.getFunction(OldFunc);
6673
6674 if (!Fn)
6675 return;
6676
6677 Function *NewFn =
6678 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6679
6680 for (User *U : make_early_inc_range(Fn->users())) {
6681 CallInst *CI = dyn_cast<CallInst>(U);
6682 if (!CI || CI->getCalledFunction() != Fn)
6683 continue;
6684
6685 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6686 FunctionType *NewFuncTy = NewFn->getFunctionType();
6687 SmallVector<Value *, 2> Args;
6688
6689 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6690 // value to the return type of the old function.
6691 if (NewFuncTy->getReturnType() != CI->getType() &&
6692 !CastInst::castIsValid(Instruction::BitCast, CI,
6693 NewFuncTy->getReturnType()))
6694 continue;
6695
6696 bool InvalidCast = false;
6697
6698 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6699 Value *Arg = CI->getArgOperand(I);
6700
6701 // Bitcast argument to the parameter type of the new function if it's
6702 // not a variadic argument.
6703 if (I < NewFuncTy->getNumParams()) {
6704 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6705 // to the parameter type of the new function.
6706 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6707 NewFuncTy->getParamType(I))) {
6708 InvalidCast = true;
6709 break;
6710 }
6711 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6712 }
6713 Args.push_back(Arg);
6714 }
6715
6716 if (InvalidCast)
6717 continue;
6718
6719 // Create a call instruction that calls the new function.
6720 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6721 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6722 NewCall->takeName(CI);
6723
6724 // Bitcast the return value back to the type of the old call.
6725 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6726
6727 if (!CI->use_empty())
6728 CI->replaceAllUsesWith(NewRetVal);
6729 CI->eraseFromParent();
6730 }
6731
6732 if (Fn->use_empty())
6733 Fn->eraseFromParent();
6734 };
6735
6736 // Unconditionally convert a call to "clang.arc.use" to a call to
6737 // "llvm.objc.clang.arc.use".
6738 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6739
6740 // Upgrade the retain release marker. If there is no need to upgrade
6741 // the marker, that means either the module is already new enough to contain
6742 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6743 if (!upgradeRetainReleaseMarker(M))
6744 return;
6745
6746 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6747 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6748 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6749 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6750 {"objc_autoreleaseReturnValue",
6751 llvm::Intrinsic::objc_autoreleaseReturnValue},
6752 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6753 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6754 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6755 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6756 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6757 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6758 {"objc_release", llvm::Intrinsic::objc_release},
6759 {"objc_retain", llvm::Intrinsic::objc_retain},
6760 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6761 {"objc_retainAutoreleaseReturnValue",
6762 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6763 {"objc_retainAutoreleasedReturnValue",
6764 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6765 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6766 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6767 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6768 {"objc_unsafeClaimAutoreleasedReturnValue",
6769 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6770 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6771 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6772 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6773 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6774 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6775 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6776 {"objc_arc_annotation_topdown_bbstart",
6777 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6778 {"objc_arc_annotation_topdown_bbend",
6779 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6780 {"objc_arc_annotation_bottomup_bbstart",
6781 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6782 {"objc_arc_annotation_bottomup_bbend",
6783 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6784
6785 for (auto &I : RuntimeFuncs)
6786 UpgradeToIntrinsic(I.first, I.second);
6787}
6788
6789// Upgrade the way signing of pointers to init/fini functions is described.
6790//
6791// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6792// constants, if signing was requested. After the upgrade, these arrays contain
6793// plain function pointers and the desired signing schema is described via a
6794// pair of module flags.
6795//
6796// Note that the upgrade is only performed if all elements of *both* arrays
6797// agree on a common signing schema.
6798static bool upgradePtrauthInitFiniArrays(Module &M) {
6799 // As we cannot always decide whether the particular module should have
6800 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6801 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6802 // returns as soon as it spots any non-signed init/fini pointer: either we
6803 // should request non-signed pointers (safe to omit both flags) or there is
6804 // no common schema (and thus we do not modify anything).
6805 //
6806 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6807 // or whether we should request address diversity in addition to the basic
6808 // constant diversity. There is no value representing "decided not to sign"
6809 // for the reasons explained above.
6810 std::optional<bool> UseAddressDisc;
6811
6812 // Do not attempt upgrading if the new module flags already exist.
6813 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6814 for (const MDNode *Flag : ModFlags->operands()) {
6815 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
6816 if (Name && (*Name == "ptrauth-init-fini" ||
6817 *Name == "ptrauth-init-fini-address-discrimination"))
6818 return false;
6819 }
6820 }
6821
6822 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6823 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6824 constexpr unsigned ExpectedAddressMarker = 1;
6825
6826 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
6827 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
6828 return nullptr; // Nothing to upgrade or unknown pattern found.
6829
6830 bool HasAddressDisc;
6831 if (!CPA->hasAddressDiscriminator())
6832 HasAddressDisc = false;
6833 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
6834 HasAddressDisc = true;
6835 else
6836 return nullptr; // Unknown pattern.
6837
6838 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6839 return nullptr; // Disagreement with the decided mode.
6840
6841 UseAddressDisc = HasAddressDisc;
6842 return CPA->getPointer();
6843 };
6844
6845 // Do not apply any changes until we know the upgrade is non-ambiguous.
6846 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6847 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6848
6849 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6850 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
6851 if (!GV || !GV->hasInitializer())
6852 continue; // Skip, but it is okay to upgrade the other variable.
6853
6854 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
6855 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6856 return false;
6857
6858 std::vector<Constant *> NewStructors;
6859 NewStructors.reserve(OldStructorsArray->getNumOperands());
6860
6861 for (Use &U : OldStructorsArray->operands()) {
6862 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
6863 if (!Structor || Structor->getNumOperands() != 3)
6864 return false;
6865
6866 Constant *Prio = Structor->getOperand(0);
6867 Constant *Func = Structor->getOperand(1);
6868 Constant *Arg = Structor->getOperand(2);
6869
6870 Func = UpgradeSinglePointer(Func);
6871 if (!Func)
6872 return false;
6873
6874 NewStructors.push_back(
6875 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
6876 }
6877
6878 Constant *NewInit =
6879 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
6880 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
6881 }
6882
6883 if (GlobalArraysToUpgrade.empty())
6884 return false;
6885 assert(UseAddressDisc.has_value());
6886
6887 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6888 GV->setInitializer(NewInit);
6889
6890 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
6891 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
6892 *UseAddressDisc);
6893
6894 return true;
6895}
6896
6897bool llvm::UpgradeModuleFlags(Module &M) {
6898 bool Changed = false;
6899 Changed |= upgradePtrauthInitFiniArrays(M);
6900
6901 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6902 if (!ModFlags)
6903 return Changed;
6904
6905 bool HasObjCFlag = false, HasClassProperties = false;
6906 bool HasSwiftVersionFlag = false;
6907 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6908 uint32_t SwiftABIVersion;
6909 auto Int8Ty = Type::getInt8Ty(M.getContext());
6910 auto Int32Ty = Type::getInt32Ty(M.getContext());
6911
6912 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6913 MDNode *Op = ModFlags->getOperand(I);
6914 if (Op->getNumOperands() != 3)
6915 continue;
6916 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6917 if (!ID)
6918 continue;
6919 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6920 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6921 Type::getInt32Ty(M.getContext()), B)),
6922 MDString::get(M.getContext(), ID->getString()),
6923 Op->getOperand(2)};
6924 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6925 Changed = true;
6926 };
6927
6928 if (ID->getString() == "Objective-C Image Info Version")
6929 HasObjCFlag = true;
6930 if (ID->getString() == "Objective-C Class Properties")
6931 HasClassProperties = true;
6932 // Upgrade PIC from Error/Max to Min.
6933 if (ID->getString() == "PIC Level") {
6934 if (auto *Behavior =
6935 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6936 uint64_t V = Behavior->getLimitedValue();
6937 if (V == Module::Error || V == Module::Max)
6938 SetBehavior(Module::Min);
6939 }
6940 }
6941 // Upgrade "PIE Level" from Error to Max.
6942 if (ID->getString() == "PIE Level")
6943 if (auto *Behavior =
6944 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
6945 if (Behavior->getLimitedValue() == Module::Error)
6946 SetBehavior(Module::Max);
6947
6948 // Upgrade branch protection and return address signing module flags. The
6949 // module flag behavior for these fields were Error and now they are Min.
6950 if (ID->getString() == "branch-target-enforcement" ||
6951 ID->getString().starts_with("sign-return-address")) {
6952 if (auto *Behavior =
6953 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6954 if (Behavior->getLimitedValue() == Module::Error) {
6955 Type *Int32Ty = Type::getInt32Ty(M.getContext());
6956 Metadata *Ops[3] = {
6957 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
6958 Op->getOperand(1), Op->getOperand(2)};
6959 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6960 Changed = true;
6961 }
6962 }
6963 }
6964
6965 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
6966 // section name so that llvm-lto will not complain about mismatching
6967 // module flags that is functionally the same.
6968 if (ID->getString() == "Objective-C Image Info Section") {
6969 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
6970 SmallVector<StringRef, 4> ValueComp;
6971 Value->getString().split(ValueComp, " ");
6972 if (ValueComp.size() != 1) {
6973 std::string NewValue;
6974 for (auto &S : ValueComp)
6975 NewValue += S.str();
6976 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
6977 MDString::get(M.getContext(), NewValue)};
6978 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6979 Changed = true;
6980 }
6981 }
6982 }
6983
6984 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
6985 // If the higher bits are set, it adds new module flag for swift info.
6986 if (ID->getString() == "Objective-C Garbage Collection") {
6987 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
6988 if (Md) {
6989 assert(Md->getValue() && "Expected non-empty metadata");
6990 auto Type = Md->getValue()->getType();
6991 if (Type == Int8Ty)
6992 continue;
6993 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
6994 if ((Val & 0xff) != Val) {
6995 HasSwiftVersionFlag = true;
6996 SwiftABIVersion = (Val & 0xff00) >> 8;
6997 SwiftMajorVersion = (Val & 0xff000000) >> 24;
6998 SwiftMinorVersion = (Val & 0xff0000) >> 16;
6999 }
7000 Metadata *Ops[3] = {
7001 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7002 Op->getOperand(1),
7003 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7004 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7005 Changed = true;
7006 }
7007 }
7008
7009 if (ID->getString() == "amdgpu_code_object_version") {
7010 Metadata *Ops[3] = {
7011 Op->getOperand(0),
7012 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7013 Op->getOperand(2)};
7014 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7015 Changed = true;
7016 }
7017
7018 // clang/PowerPC used to use "float-abi" to describe the long double format;
7019 // it has been renamed to "long-double-type", with its values changed to the
7020 // corresponding IR floating-point type names.
7021 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7022 StringRef Format;
7023 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7024 Format = S->getString();
7025
7026 // The "float-abi" key is now reserved for the target-independent
7027 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7028 // (including unrecognized ones, which were never valid) to the default.
7029 if (!FloatABI::parseABIType(Format)) {
7030 LongDoubleFormat NewFormat =
7031 StringSwitch<LongDoubleFormat>(Format)
7032 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7033 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7034 .Default(LongDoubleFormat::PPCDoubleDouble);
7035 Metadata *Ops[3] = {
7036 Op->getOperand(0),
7037 MDString::get(M.getContext(), "long-double-type"),
7038 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7039 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7040 Changed = true;
7041 }
7042 }
7043 }
7044
7045 // "Objective-C Class Properties" is recently added for Objective-C. We
7046 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7047 // flag of value 0, so we can correclty downgrade this flag when trying to
7048 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7049 // this module flag.
7050 if (HasObjCFlag && !HasClassProperties) {
7051 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7052 (uint32_t)0);
7053 Changed = true;
7054 }
7055
7056 if (HasSwiftVersionFlag) {
7057 M.addModuleFlag(Module::Error, "Swift ABI Version",
7058 SwiftABIVersion);
7059 M.addModuleFlag(Module::Error, "Swift Major Version",
7060 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7061 M.addModuleFlag(Module::Error, "Swift Minor Version",
7062 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7063 Changed = true;
7064 }
7065
7066 return Changed;
7067}
7068
7069bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7070 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7071 // If this metadata has operands, we expect all of them to be either from
7072 // before or from after the format change handled here, so we can bail out
7073 // fast if the first (if any) operands is of the new format.
7074 auto MatchesVersion = [](const MDNode *Op) {
7075 return Op->getNumOperands() >= 3 &&
7076 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7077 cast<ConstantAsMetadata>(Op->getOperand(2))
7078 ->getType()
7079 ->isIntegerTy(64);
7080 };
7081
7082 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7083 MatchesVersion(CFIConsts->getOperand(0)))
7084 return false;
7085
7086 bool Changed = false;
7087 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7088 MDNode *Op = CFIConsts->getOperand(I);
7089 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7090 assert(Op->getNumOperands() >= 2 &&
7091 "Expected at least 2 operands - name and linkage type");
7092 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7093 StringRef Name = NameMD->getString();
7094 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7095 GlobalValue::dropLLVMManglingEscape(Name));
7096
7097 SmallVector<Metadata *, 4> Elts;
7098 Elts.push_back(Op->getOperand(0));
7099 Elts.push_back(Op->getOperand(1));
7100 Elts.push_back(ConstantAsMetadata::get(
7101 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7102
7103 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7104 Elts.push_back(Op->getOperand(J));
7105
7106 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7107 Changed = true;
7108 }
7109
7110 return Changed;
7111}
7112
7113void llvm::UpgradeSectionAttributes(Module &M) {
7114 auto TrimSpaces = [](StringRef Section) -> std::string {
7115 SmallVector<StringRef, 5> Components;
7116 Section.split(Components, ',');
7117
7118 SmallString<32> Buffer;
7119 raw_svector_ostream OS(Buffer);
7120
7121 for (auto Component : Components)
7122 OS << ',' << Component.trim();
7123
7124 return std::string(OS.str().substr(1));
7125 };
7126
7127 for (auto &GV : M.globals()) {
7128 if (!GV.hasSection())
7129 continue;
7130
7131 StringRef Section = GV.getSection();
7132
7133 if (!Section.starts_with("__DATA, __objc_catlist"))
7134 continue;
7135
7136 // __DATA, __objc_catlist, regular, no_dead_strip
7137 // __DATA,__objc_catlist,regular,no_dead_strip
7138 GV.setSection(TrimSpaces(Section));
7139 }
7140}
7141
7142namespace {
7143// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7144// callsites within a function that did not also have the strictfp attribute.
7145// Since 10.0, if strict FP semantics are needed within a function, the
7146// function must have the strictfp attribute and all calls within the function
7147// must also have the strictfp attribute. This latter restriction is
7148// necessary to prevent unwanted libcall simplification when a function is
7149// being cloned (such as for inlining).
7150//
7151// The "dangling" strictfp attribute usage was only used to prevent constant
7152// folding and other libcall simplification. The nobuiltin attribute on the
7153// callsite has the same effect.
7154struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7155 StrictFPUpgradeVisitor() = default;
7156
7157 void visitCallBase(CallBase &Call) {
7158 if (!Call.isStrictFP())
7159 return;
7160 if (isa<ConstrainedFPIntrinsic>(&Call))
7161 return;
7162 // If we get here, the caller doesn't have the strictfp attribute
7163 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7164 Call.removeFnAttr(Attribute::StrictFP);
7165 Call.addFnAttr(Attribute::NoBuiltin);
7166 }
7167};
7168
7169/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7170struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7171 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7172 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7173
7174 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7175 if (!RMW.isFloatingPointOperation())
7176 return;
7177
7178 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7179 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7180 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7181 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
7182 }
7183};
7184} // namespace
7185
7186void llvm::UpgradeFunctionAttributes(Function &F) {
7187 // If a function definition doesn't have the strictfp attribute,
7188 // convert any callsite strictfp attributes to nobuiltin.
7189 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7190 StrictFPUpgradeVisitor SFPV;
7191 SFPV.visit(F);
7192 }
7193
7194 // Remove all incompatibile attributes from function.
7195 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7196 F.getReturnType(), F.getAttributes().getRetAttrs()));
7197 for (auto &Arg : F.args())
7198 Arg.removeAttrs(
7199 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7200
7201 bool AddingAttrs = false, RemovingAttrs = false;
7202 AttrBuilder AttrsToAdd(F.getContext());
7203 AttributeMask AttrsToRemove;
7204
7205 // Older versions of LLVM treated an "implicit-section-name" attribute
7206 // similarly to directly setting the section on a Function.
7207 if (Attribute A = F.getFnAttribute("implicit-section-name");
7208 A.isValid() && A.isStringAttribute()) {
7209 F.setSection(A.getValueAsString());
7210 AttrsToRemove.addAttribute("implicit-section-name");
7211 RemovingAttrs = true;
7212 }
7213
7214 if (Attribute A = F.getFnAttribute("nooutline");
7215 A.isValid() && A.isStringAttribute()) {
7216 AttrsToRemove.addAttribute("nooutline");
7217 AttrsToAdd.addAttribute(Attribute::NoOutline);
7218 AddingAttrs = RemovingAttrs = true;
7219 }
7220
7221 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7222 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7223 AttrsToRemove.addAttribute("uniform-work-group-size");
7224 RemovingAttrs = true;
7225 if (A.getValueAsString() == "true") {
7226 AttrsToAdd.addAttribute("uniform-work-group-size");
7227 AddingAttrs = true;
7228 }
7229 }
7230
7231 if (!F.empty()) {
7232 // For some reason this is called twice, and the first time is before any
7233 // instructions are loaded into the body.
7234
7235 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7236 A.isValid()) {
7237
7238 if (A.getValueAsBool()) {
7239 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7240 Visitor.visit(F);
7241 }
7242
7243 // We will leave behind dead attribute uses on external declarations, but
7244 // clang never added these to declarations anyway.
7245 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7246 RemovingAttrs = true;
7247 }
7248 }
7249
7250 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7251 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7252
7253 bool HandleDenormalMode = false;
7254
7255 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7256 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7257 if (ParsedMode.isValid()) {
7258 DenormalFPMath = ParsedMode;
7259 AttrsToRemove.addAttribute("denormal-fp-math");
7260 AddingAttrs = RemovingAttrs = true;
7261 HandleDenormalMode = true;
7262 }
7263 }
7264
7265 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7266 Attr.isValid()) {
7267 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7268 if (ParsedMode.isValid()) {
7269 DenormalFPMathF32 = ParsedMode;
7270 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7271 AddingAttrs = RemovingAttrs = true;
7272 HandleDenormalMode = true;
7273 }
7274 }
7275
7276 if (HandleDenormalMode)
7277 AttrsToAdd.addDenormalFPEnvAttr(
7278 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7279
7280 if (RemovingAttrs)
7281 F.removeFnAttrs(AttrsToRemove);
7282
7283 if (AddingAttrs)
7284 F.addFnAttrs(AttrsToAdd);
7285}
7286
7287// Check if the function attribute is not present and set it.
7288static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7289 StringRef Value) {
7290 if (!F.hasFnAttribute(FnAttrName))
7291 F.addFnAttr(FnAttrName, Value);
7292}
7293
7294// Check if the function attribute is not present and set it if needed.
7295// If the attribute is "false" then removes it.
7296// If the attribute is "true" resets it to a valueless attribute.
7297static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7298 if (!F.hasFnAttribute(FnAttrName)) {
7299 if (Set)
7300 F.addFnAttr(FnAttrName);
7301 } else {
7302 auto A = F.getFnAttribute(FnAttrName);
7303 if ("false" == A.getValueAsString())
7304 F.removeFnAttr(FnAttrName);
7305 else if ("true" == A.getValueAsString()) {
7306 F.removeFnAttr(FnAttrName);
7307 F.addFnAttr(FnAttrName);
7308 }
7309 }
7310}
7311
7312void llvm::copyModuleAttrToFunctions(Module &M) {
7313 Triple T(M.getTargetTriple());
7314 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7315 return;
7316
7317 uint64_t BTEValue = 0;
7318 uint64_t BPPLRValue = 0;
7319 uint64_t GCSValue = 0;
7320 uint64_t SRAValue = 0;
7321 uint64_t SRAALLValue = 0;
7322 uint64_t SRABKeyValue = 0;
7323
7324 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7325 if (ModFlags) {
7326 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7327 MDNode *Op = ModFlags->getOperand(I);
7328 if (Op->getNumOperands() != 3)
7329 continue;
7330
7331 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7332 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7333 if (!ID || !CI)
7334 continue;
7335
7336 StringRef IDStr = ID->getString();
7337 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7338 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7339 : IDStr == "guarded-control-stack" ? &GCSValue
7340 : IDStr == "sign-return-address" ? &SRAValue
7341 : IDStr == "sign-return-address-all" ? &SRAALLValue
7342 : IDStr == "sign-return-address-with-bkey"
7343 ? &SRABKeyValue
7344 : nullptr;
7345 if (!ValPtr)
7346 continue;
7347
7348 *ValPtr = CI->getZExtValue();
7349 if (*ValPtr == 2)
7350 return;
7351 }
7352 }
7353
7354 bool BTE = BTEValue == 1;
7355 bool BPPLR = BPPLRValue == 1;
7356 bool GCS = GCSValue == 1;
7357 bool SRA = SRAValue == 1;
7358
7359 StringRef SignTypeValue = "non-leaf";
7360 if (SRA && SRAALLValue == 1)
7361 SignTypeValue = "all";
7362
7363 StringRef SignKeyValue = "a_key";
7364 if (SRA && SRABKeyValue == 1)
7365 SignKeyValue = "b_key";
7366
7367 for (Function &F : M.getFunctionList()) {
7368 if (F.isDeclaration())
7369 continue;
7370
7371 if (SRA) {
7372 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7373 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7374 } else {
7375 if (auto A = F.getFnAttribute("sign-return-address");
7376 A.isValid() && "none" == A.getValueAsString()) {
7377 F.removeFnAttr("sign-return-address");
7378 F.removeFnAttr("sign-return-address-key");
7379 }
7380 }
7381 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7382 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7383 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7384 }
7385
7386 if (BTE)
7387 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7388 if (BPPLR)
7389 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7390 if (GCS)
7391 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7392 if (SRA) {
7393 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7394 if (SRAALLValue == 1)
7395 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7396 if (SRABKeyValue == 1)
7397 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7398 }
7399}
7400
7401/// Return the replacement tags if \p T still uses a removed two-operand form.
7402static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7403 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7404 return nullptr;
7405 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7406 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7407}
7408
7409/// Build the single-operand node that replaces a boolean operand: nonzero
7410/// selects the enable tag, zero the disable tag.
7411static Metadata *makeBooleanLoopNode(LLVMContext &C,
7412 const BooleanLoopTags &Tags,
7413 const MDOperand &Op) {
7414 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7415 return MDTuple::get(C,
7416 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7417}
7418
7419static bool isOldLoopArgument(Metadata *MD) {
7420 auto *T = dyn_cast_or_null<MDTuple>(MD);
7421 if (!T)
7422 return false;
7423 if (T->getNumOperands() < 1)
7424 return false;
7425 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7426 if (!S)
7427 return false;
7428 if (S->getString().starts_with("llvm.vectorizer."))
7429 return true;
7430 return getOldBooleanLoopTags(T) != nullptr;
7431}
7432
7433static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7434 StringRef OldPrefix = "llvm.vectorizer.";
7435 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7436
7437 if (OldTag == "llvm.vectorizer.unroll")
7438 return MDString::get(C, "llvm.loop.interleave.count");
7439
7440 return MDString::get(
7441 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7442 .str());
7443}
7444
7445static Metadata *upgradeLoopArgument(Metadata *MD) {
7446 auto *T = dyn_cast_or_null<MDTuple>(MD);
7447 if (!T)
7448 return MD;
7449 if (T->getNumOperands() < 1)
7450 return MD;
7451 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7452 if (!OldTag)
7453 return MD;
7454
7455 LLVMContext &C = T->getContext();
7456
7457 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7458 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7459 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7460
7461 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7462 return MD;
7463
7464 // This has an old tag. Upgrade it.
7465 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7466
7467 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7468 // vectorize.enable/disable pair, not a two-operand enable node.
7469 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7470 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7471 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7472
7473 SmallVector<Metadata *, 8> Ops;
7474 Ops.reserve(T->getNumOperands());
7475 Ops.push_back(NewTag);
7476 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7477 Ops.push_back(T->getOperand(I));
7478
7479 return MDTuple::get(C, Ops);
7480}
7481
7482MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7483 auto *T = dyn_cast<MDTuple>(&N);
7484 if (!T)
7485 return &N;
7486
7487 if (none_of(T->operands(), isOldLoopArgument))
7488 return &N;
7489
7490 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7491 // any MDNode carrying those tags with more than one operand, so a leftover
7492 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7493 // In-place mutation is safe on distinct MDNodes.
7494 if (T->isDistinct()) {
7495 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7496 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7497 if (OpT && getOldBooleanLoopTags(OpT))
7498 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7499 }
7500 if (none_of(T->operands(), isOldLoopArgument))
7501 return &N;
7502 }
7503
7504 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7505 // attachment; the original distinct loop-ID is kept as the first operand.
7506 SmallVector<Metadata *, 8> Ops;
7507 Ops.reserve(T->getNumOperands());
7508 for (Metadata *MD : T->operands())
7509 Ops.push_back(upgradeLoopArgument(MD));
7510
7511 return MDTuple::get(T->getContext(), Ops);
7512}
7513
7514std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7515 Triple T(TT);
7516 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7517 // the address space of globals to 1. This does not apply to SPIRV Logical.
7518 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7519 !DL.contains("-G") && !DL.starts_with("G")) {
7520 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7521 }
7522
7523 if (T.isLoongArch64() || T.isRISCV64()) {
7524 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7525 auto I = DL.find("-n64-");
7526 if (I != StringRef::npos)
7527 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7528 return DL.str();
7529 }
7530
7531 // AMDGPU data layout upgrades.
7532 std::string Res = DL.str();
7533 if (T.isAMDGPU()) {
7534 // Define address spaces for constants.
7535 if (!DL.contains("-G") && !DL.starts_with("G"))
7536 Res.append(Res.empty() ? "G1" : "-G1");
7537
7538 // AMDGCN data layout upgrades.
7539 if (T.isAMDGCN()) {
7540
7541 // Add missing non-integral declarations.
7542 // This goes before adding new address spaces to prevent incoherent string
7543 // values.
7544 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7545 Res.append("-ni:7:8:9");
7546 // Update ni:7 to ni:7:8:9.
7547 if (DL.ends_with("ni:7"))
7548 Res.append(":8:9");
7549 if (DL.ends_with("ni:7:8"))
7550 Res.append(":9");
7551
7552 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7553 // resources) An empty data layout has already been upgraded to G1 by now.
7554 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7555 Res.append("-p7:160:256:256:32");
7556 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7557 Res.append("-p8:128:128:128:48");
7558 constexpr StringRef OldP8("-p8:128:128-");
7559 if (DL.contains(OldP8))
7560 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7561 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7562 Res.append("-p9:192:256:256:32");
7563 }
7564
7565 // Upgrade the ELF mangling mode.
7566 if (!DL.contains("m:e"))
7567 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7568
7569 return Res;
7570 }
7571
7572 if (T.isSystemZ() && !DL.empty()) {
7573 // Make sure the stack alignment is present.
7574 if (!DL.contains("-S64"))
7575 return "E-S64" + DL.drop_front(1).str();
7576 return DL.str();
7577 }
7578
7579 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7580 // If the datalayout matches the expected format, add pointer size address
7581 // spaces to the datalayout.
7582 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7583 if (!DL.contains(AddrSpaces)) {
7584 SmallVector<StringRef, 4> Groups;
7585 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7586 if (R.match(Res, &Groups))
7587 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7588 }
7589 };
7590
7591 // AArch64 data layout upgrades.
7592 if (T.isAArch64()) {
7593 // Add "-Fn32"
7594 if (!DL.empty() && !DL.contains("-Fn32"))
7595 Res.append("-Fn32");
7596 AddPtr32Ptr64AddrSpaces();
7597 return Res;
7598 }
7599
7600 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7601 T.isWasm()) {
7602 // Mips64 with o32 ABI did not add "-i128:128".
7603 // Add "-i128:128"
7604 std::string I64 = "-i64:64";
7605 std::string I128 = "-i128:128";
7606 if (!StringRef(Res).contains(I128)) {
7607 size_t Pos = Res.find(I64);
7608 if (Pos != size_t(-1))
7609 Res.insert(Pos + I64.size(), I128);
7610 }
7611 }
7612
7613 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7614 size_t Pos = Res.find("-S128");
7615 if (Pos == StringRef::npos)
7616 Pos = Res.size();
7617 Res.insert(Pos, "-f64:32:64");
7618 }
7619
7620 if (!T.isX86())
7621 return Res;
7622
7623 AddPtr32Ptr64AddrSpaces();
7624
7625 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7626 // for i128 operations prior to this being reflected in the data layout, and
7627 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7628 // boundaries, so although this is a breaking change, the upgrade is expected
7629 // to fix more IR than it breaks.
7630 // Intel MCU is an exception and uses 4-byte-alignment.
7631 if (!T.isOSIAMCU()) {
7632 std::string I128 = "-i128:128";
7633 if (StringRef Ref = Res; !Ref.contains(I128)) {
7634 SmallVector<StringRef, 4> Groups;
7635 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7636 if (R.match(Res, &Groups))
7637 Res = (Groups[1] + I128 + Groups[3]).str();
7638 }
7639 }
7640
7641 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7642 // Raising the alignment is safe because Clang did not produce f80 values in
7643 // the MSVC environment before this upgrade was added.
7644 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7645 StringRef Ref = Res;
7646 auto I = Ref.find("-f80:32-");
7647 if (I != StringRef::npos)
7648 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7649 }
7650
7651 return Res;
7652}
7653
7654void llvm::UpgradeAttributes(AttrBuilder &B) {
7655 StringRef FramePointer;
7656 Attribute A = B.getAttribute("no-frame-pointer-elim");
7657 if (A.isValid()) {
7658 // The value can be "true" or "false".
7659 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7660 B.removeAttribute("no-frame-pointer-elim");
7661 }
7662 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7663 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7664 if (FramePointer != "all")
7665 FramePointer = "non-leaf";
7666 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7667 }
7668 if (!FramePointer.empty())
7669 B.addAttribute("frame-pointer", FramePointer);
7670
7671 A = B.getAttribute("null-pointer-is-valid");
7672 if (A.isValid()) {
7673 // The value can be "true" or "false".
7674 bool NullPointerIsValid = A.getValueAsString() == "true";
7675 B.removeAttribute("null-pointer-is-valid");
7676 if (NullPointerIsValid)
7677 B.addAttribute(Attribute::NullPointerIsValid);
7678 }
7679
7680 A = B.getAttribute("uniform-work-group-size");
7681 if (A.isValid()) {
7682 StringRef Val = A.getValueAsString();
7683 if (!Val.empty()) {
7684 bool IsTrue = Val == "true";
7685 B.removeAttribute("uniform-work-group-size");
7686 if (IsTrue)
7687 B.addAttribute("uniform-work-group-size");
7688 }
7689 }
7690}
7691
7692void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7693 // clang.arc.attachedcall bundles are now required to have an operand.
7694 // If they don't, it's okay to drop them entirely: when there is an operand,
7695 // the "attachedcall" is meaningful and required, but without an operand,
7696 // it's just a marker NOP. Dropping it merely prevents an optimization.
7697 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7698 return OBD.getTag() == "clang.arc.attachedcall" &&
7699 OBD.inputs().empty();
7700 });
7701}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:482
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:483
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1491
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2434
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7445
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6537
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1521
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2274
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1232
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5389
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1199
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1212
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6643
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2535
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2761
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1471
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2509
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1424
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7288
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1303
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6541
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5249
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7433
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5349
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6504
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2389
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2646
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1467
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2360
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4908
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2747
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7402
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2686
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1283
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6450
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1381
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7411
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2308
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1387
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1258
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1149
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7419
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4978
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5319
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5068
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2625
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5261
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5420
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2340
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2372
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2574
shouldUpgradeNVPTXTcgen05MMAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1369
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2708
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2494
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3018
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2740
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2770
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5271
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7297
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2656
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2607
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5332
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6798
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3216
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5367
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:769
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:929
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:781
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:785
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:806
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:845
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:799
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:789
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:795
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:849
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:837
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:797
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:817
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:813
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:841
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:976
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:507
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:355
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1590
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1710
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3105
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3388
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3473
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:168
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:211
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:246
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:166
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:448
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:885
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:216
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:870
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2893
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:321
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:519
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:317
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1792
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:700
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:316
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1500
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:310
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:190
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:633
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:615
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:111
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:117
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:138
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:120
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:152
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:149
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1576
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1568
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1564
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1583
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1574
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2090
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:386
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:532
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:977
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:702
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1236
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:331
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:823
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5264
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2356
Changed
Changed
Definition ObjCARCOpts.cpp:2366
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1418
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:758
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1399
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1392
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:578
llvm::Length
@ Length
Definition DWP.cpp:578
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5475
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7113
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3009
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2222
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7482
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7654
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6363
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6602
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6897
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7069
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7312
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7692
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6430
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7514
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2241
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:709
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:322
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6408
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6460
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7186
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6382
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6667
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7685
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0