LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front("addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front("ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(1)->getType();
1095 Type *Ty = VectorType::get(ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front("tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with("get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 F->getParent(), Intrinsic::vector_extract, Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with("set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 F->getParent(), Intrinsic::vector_insert, Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 F->getParent(), Intrinsic::vector_insert, Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with("rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front("sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front("ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1172// declarations requiring an auto-upgrade. The same set applies to the
1173// cluster (g2s) and CTA (g2s_cta) variants.
1174#define NVVM_TMA_G2S_MODES(M) \
1175 M(tile_1d, "tile.1d") \
1176 M(tile_2d, "tile.2d") \
1177 M(tile_3d, "tile.3d") \
1178 M(tile_4d, "tile.4d") \
1179 M(tile_5d, "tile.5d") \
1180 M(tile_gather4_2d, "tile.gather4.2d") \
1181 M(im2col_3d, "im2col.3d") \
1182 M(im2col_4d, "im2col.4d") \
1183 M(im2col_5d, "im2col.5d") \
1184 M(im2col_w_3d, "im2col.w.3d") \
1185 M(im2col_w_4d, "im2col.w.4d") \
1186 M(im2col_w_5d, "im2col.w.5d") \
1187 M(im2col_w_128_3d, "im2col.w.128.3d") \
1188 M(im2col_w_128_4d, "im2col.w.128.4d") \
1189 M(im2col_w_128_5d, "im2col.w.128.5d")
1190
1191// Two legacy tails are:
1192//
1193// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1194// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1195//
1196// The current tail appends a trailing i32 %validate_pattern, so both
1197// legacy tails are recognized by an i1 at parameter N-2.
1198static Intrinsic::ID
1199shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1200 SmallVectorImpl<Type *> &OvlTys) {
1201 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1202 return Intrinsic::not_intrinsic;
1203
1204#define G2S_ID(ID_SUFFIX, NAME) \
1205 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1206 // clang-format off
1207 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1208 NVVM_TMA_G2S_MODES(G2S_ID)
1209 .Default(Intrinsic::not_intrinsic);
1210#undef G2S_ID
1211 // clang-format on
1212 if (ID == Intrinsic::not_intrinsic)
1213 return ID;
1214
1215 size_t NumParams = F->getFunctionType()->getNumParams();
1216
1217 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1218 // with i32 %cta_group, i32 %validate_pattern, for which N-2 is i32.
1219 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1220 return Intrinsic::not_intrinsic;
1221
1222 // The multicast mask is the parameter immediately before the i64
1223 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1224 ArrayRef<Type *> Params = F->getFunctionType()->params();
1225 size_t MaskIdx =
1226 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1227 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1228 "expected the i64 cache-hint after the multicast mask");
1229 Type *MaskTy = Params[MaskIdx];
1230 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1231 OvlTys.push_back(MaskTy);
1232
1233 return ID;
1234}
1235
1236// The legacy tail is:
1237//
1238// arg1, arg2, .. i64 %ch, i1 %flag_ch
1239//
1240// The current tail appends a trailing i32 %validate_pattern, so the
1241// legacy tail is recognized by an i1 at parameter N-1.
1242static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1243 StringRef Name) {
1244 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1245 return Intrinsic::not_intrinsic;
1246
1247#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1248 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1249 // clang-format off
1250 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1251 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1252 .Default(Intrinsic::not_intrinsic);
1253#undef G2S_CTA_ID
1254 // clang-format on
1255 if (ID == Intrinsic::not_intrinsic)
1256 return ID;
1257
1258 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1259 // with i32 %validate_pattern, for which N-1 is i32.
1260 if (!F->getFunctionType()
1261 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1262 ->isIntegerTy(1))
1263 return Intrinsic::not_intrinsic;
1264
1265 return ID;
1266}
1267// The legacy TMA reduction intrinsics encode the reduction operator in their
1268// name, while the current ones take it as an immediate argument. Map the
1269// operator part of a legacy name to the corresponding immediate value.
1270static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1271 return StringSwitch<std::optional<unsigned>>(Name)
1272 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1273 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1274 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1275 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1276 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1277 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1278 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1279 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1280 .Default(std::nullopt);
1281}
1282
1283static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1284 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1285 return Intrinsic::not_intrinsic;
1286
1287 auto [RedOpName, ShapeName] = Name.split('.');
1288 if (!getNVPTXTMAReductionOp(RedOpName))
1289 return Intrinsic::not_intrinsic;
1290
1291 return StringSwitch<Intrinsic::ID>(ShapeName)
1292 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1293 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1294 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1295 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1296 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1297 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1298 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1299 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1300 .Default(Intrinsic::not_intrinsic);
1301}
1302
1303static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1304 StringRef Name) {
1305 if (Name.consume_front("mapa.shared.cluster"))
1306 if (F->getReturnType()->getPointerAddressSpace() ==
1307 NVPTXAS::ADDRESS_SPACE_SHARED)
1308 return Intrinsic::nvvm_mapa_shared_cluster;
1309
1310 if (Name.consume_front("cp.async.bulk.")) {
1311 Intrinsic::ID ID =
1312 StringSwitch<Intrinsic::ID>(Name)
1313 .Case("global.to.shared.cluster",
1314 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1315 .Case("shared.cta.to.cluster",
1316 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1317 .Default(Intrinsic::not_intrinsic);
1318
1319 if (ID != Intrinsic::not_intrinsic)
1320 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1321 NVPTXAS::ADDRESS_SPACE_SHARED)
1322 return ID;
1323 }
1324
1325 return Intrinsic::not_intrinsic;
1326}
1327
1328static Intrinsic::ID
1329shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1330 if (!Name.consume_front("tcgen05.commit."))
1331 return Intrinsic::not_intrinsic;
1332
1333 if (Name.consume_front("shared."))
1334 return StringSwitch<Intrinsic::ID>(Name)
1335 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1336 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1337 .Default(Intrinsic::not_intrinsic);
1338
1339 if (Name.consume_front("mc.shared.")) {
1340 // Only upgrade older i16 mc variants.
1341 if (!F->getArg(1)->getType()->isIntegerTy(16))
1342 return Intrinsic::not_intrinsic;
1343
1344 return StringSwitch<Intrinsic::ID>(Name)
1345 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1346 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1347 .Default(Intrinsic::not_intrinsic);
1348 }
1349
1350 return Intrinsic::not_intrinsic;
1351}
1352
1353static Intrinsic::ID
1354shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1355 if (F->arg_size() != 2)
1356 return Intrinsic::not_intrinsic;
1357
1358 if (Name.consume_front("tcgen05.alloc.shared.") ||
1359 Name.consume_front("tcgen05.alloc."))
1360 return StringSwitch<Intrinsic::ID>(Name)
1361 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1362 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1363 .Default(Intrinsic::not_intrinsic);
1364
1365 if (Name.consume_front("tcgen05.dealloc."))
1366 return StringSwitch<Intrinsic::ID>(Name)
1367 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1368 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1369 .Default(Intrinsic::not_intrinsic);
1370
1371 return Intrinsic::not_intrinsic;
1372}
1373
1374static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1375 if (Name.consume_front("fma.rn."))
1376 return StringSwitch<Intrinsic::ID>(Name)
1377 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1378 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1379 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1380 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1381 .Default(Intrinsic::not_intrinsic);
1382
1383 if (Name.consume_front("fmax."))
1384 return StringSwitch<Intrinsic::ID>(Name)
1385 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1386 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1387 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1388 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1389 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1390 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1391 .Case("ftz.nan.xorsign.abs.bf16",
1392 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1393 .Case("ftz.nan.xorsign.abs.bf16x2",
1394 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1395 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1396 .Case("ftz.xorsign.abs.bf16x2",
1397 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1398 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1399 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1400 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1401 .Case("nan.xorsign.abs.bf16x2",
1402 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1403 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1404 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1405 .Default(Intrinsic::not_intrinsic);
1406
1407 if (Name.consume_front("fmin."))
1408 return StringSwitch<Intrinsic::ID>(Name)
1409 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1410 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1411 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1412 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1413 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1414 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1415 .Case("ftz.nan.xorsign.abs.bf16",
1416 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1417 .Case("ftz.nan.xorsign.abs.bf16x2",
1418 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1419 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1420 .Case("ftz.xorsign.abs.bf16x2",
1421 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1422 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1423 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1424 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1425 .Case("nan.xorsign.abs.bf16x2",
1426 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1427 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1428 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1429 .Default(Intrinsic::not_intrinsic);
1430
1431 if (Name.consume_front("neg."))
1432 return StringSwitch<Intrinsic::ID>(Name)
1433 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1434 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1435 .Default(Intrinsic::not_intrinsic);
1436
1437 return Intrinsic::not_intrinsic;
1438}
1439
1440static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1441 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1442 FunctionType *OldFnTy = F->getFunctionType();
1443 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1444 return OldTy->getScalarType()->isIntegerTy() &&
1445 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1446 };
1447
1448 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1449 return false;
1450
1451 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1452 return false;
1453
1454 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1455 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1456 return false;
1457
1458 return true;
1459}
1460
1461static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1462 StringRef Name) {
1463 if (!Name.consume_front("tcgen05.mma."))
1464 return Intrinsic::not_intrinsic;
1465
1466 // tcgen05.mma.ws.* variants do not need collector-b appended.
1467 if (Name.starts_with("ws"))
1468 return Intrinsic::not_intrinsic;
1469
1470 return F->getIntrinsicID();
1471}
1472
1473static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1474getNVVMFAddUpgrade(StringRef Name) {
1475 auto [Modifiers, Type] = Name.rsplit('.');
1476 if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
1477 return std::nullopt;
1478
1479 std::optional<llvm::RoundingMode> RoundingMode =
1480 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1481 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1482 .Case("rz", llvm::RoundingMode::TowardZero)
1483 .Case("rm", llvm::RoundingMode::TowardNegative)
1484 .Case("rp", llvm::RoundingMode::TowardPositive)
1485 .Default(std::nullopt);
1486 if (!RoundingMode)
1487 return std::nullopt;
1488
1489 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
1490 .Case("", Intrinsic::nvvm_fadd)
1491 .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
1492 .Case(".sat", Intrinsic::nvvm_fadd_sat)
1493 .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
1494 .Default(Intrinsic::not_intrinsic);
1495 if (IID == Intrinsic::not_intrinsic)
1496 return std::nullopt;
1497
1498 return std::make_pair(IID, *RoundingMode);
1499}
1500
1501static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1502 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1503 return Intrinsic::not_intrinsic;
1504
1505 return Intrinsic::nvvm_mbarrier_init;
1506}
1507
1508static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1509 return Name.consume_front("local") || Name.consume_front("shared") ||
1510 Name.consume_front("global") || Name.consume_front("constant") ||
1511 Name.consume_front("param");
1512}
1513
1514static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1515 if (!Name.consume_front("vp."))
1516 return 0;
1517 return StringSwitch<unsigned>(Name)
1518 .StartsWith("select", Instruction::Select)
1519 .StartsWith("add", Instruction::Add)
1520 .StartsWith("sub", Instruction::Sub)
1521 .StartsWith("mul", Instruction::Mul)
1522 .StartsWith("ashr", Instruction::AShr)
1523 .StartsWith("lshr", Instruction::LShr)
1524 .StartsWith("shl", Instruction::Shl)
1525 .StartsWith("or", Instruction::Or)
1526 .StartsWith("and", Instruction::And)
1527 .StartsWith("xor", Instruction::Xor)
1528 .StartsWith("fadd", Instruction::FAdd)
1529 .StartsWith("fsub", Instruction::FSub)
1530 .StartsWith("fmuladd", 0)
1531 .StartsWith("fmul", Instruction::FMul)
1532 .StartsWith("fdiv", Instruction::FDiv)
1533 .StartsWith("frem", Instruction::FRem)
1534 .StartsWith("fneg", Instruction::FNeg)
1535 .StartsWith("trunc", Instruction::Trunc)
1536 .StartsWith("zext", Instruction::ZExt)
1537 .StartsWith("sext", Instruction::SExt)
1538 .StartsWith("fptrunc", Instruction::FPTrunc)
1539 .StartsWith("fpext", Instruction::FPExt)
1540 .StartsWith("fptoui", Instruction::FPToUI)
1541 .StartsWith("fptosi", Instruction::FPToSI)
1542 .StartsWith("uitofp", Instruction::UIToFP)
1543 .StartsWith("sitofp", Instruction::SIToFP)
1544 .StartsWith("ptrtoint", Instruction::PtrToInt)
1545 .StartsWith("inttoptr", Instruction::IntToPtr)
1546 .StartsWith("icmp", Instruction::ICmp)
1547 .StartsWith("fcmp", Instruction::FCmp)
1548 .Default(0);
1549}
1550
1551static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1552 if (!Name.consume_front("vp."))
1553 return 0;
1554 return StringSwitch<Intrinsic::ID>(Name)
1555 .StartsWith("abs", Intrinsic::abs)
1556 .StartsWith("smax", Intrinsic::smax)
1557 .StartsWith("smin", Intrinsic::smin)
1558 .StartsWith("umax", Intrinsic::umax)
1559 .StartsWith("umin", Intrinsic::umin)
1560 .StartsWith("copysign", Intrinsic::copysign)
1561 .StartsWith("minnum", Intrinsic::minnum)
1562 .StartsWith("maxnum", Intrinsic::maxnum)
1563 .StartsWith("minimum", Intrinsic::minimum)
1564 .StartsWith("maximum", Intrinsic::maximum)
1565 .StartsWith("fabs", Intrinsic::fabs)
1566 .StartsWith("sqrt", Intrinsic::sqrt)
1567 .StartsWith("fma", Intrinsic::fma)
1568 .StartsWith("fmuladd", Intrinsic::fmuladd)
1569 .StartsWith("ceil", Intrinsic::ceil)
1570 .StartsWith("floor", Intrinsic::floor)
1571 .StartsWith("rint", Intrinsic::rint)
1572 .StartsWith("nearbyint", Intrinsic::nearbyint)
1573 .StartsWith("roundeven", Intrinsic::roundeven)
1574 .StartsWith("roundtozero", Intrinsic::trunc)
1575 .StartsWith("round", Intrinsic::round)
1576 .StartsWith("lrint", Intrinsic::lrint)
1577 .StartsWith("llrint", Intrinsic::llrint)
1578 .StartsWith("bitreverse", Intrinsic::bitreverse)
1579 .StartsWith("bswap", Intrinsic::bswap)
1580 .StartsWith("ctpop", Intrinsic::ctpop)
1581 .StartsWith("ctlz", Intrinsic::ctlz)
1582 .StartsWith("cttz.elts", 0)
1583 .StartsWith("cttz", Intrinsic::cttz)
1584 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1585 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1586 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1587 .StartsWith("usub.sat", Intrinsic::usub_sat)
1588 .StartsWith("fshl", Intrinsic::fshl)
1589 .StartsWith("fshr", Intrinsic::fshr)
1590 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1591 .Default(0);
1592}
1593
1597
1598static bool convertIntrinsicValidType(StringRef Name,
1599 const FunctionType *FuncTy) {
1600 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1601 if (Name.starts_with("to.fp16")) {
1602 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1603 HalfTy) &&
1604 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1605 FuncTy->getReturnType());
1606 }
1607
1608 if (Name.starts_with("from.fp16")) {
1609 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1610 HalfTy) &&
1611 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1612 FuncTy->getReturnType());
1613 }
1614
1615 return false;
1616}
1617
1618static unsigned getFullArgCountForDefaultArgUpgrade(Function *F,
1619 Intrinsic::ID IID) {
1620 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1621 if (Defaults.empty())
1622 return 0;
1623
1624 if (Intrinsic::isOverloaded(IID))
1625 return 0;
1626
1627 unsigned FullArgCount = FirstDefault + Defaults.size();
1628
1629 // Only trailing default arguments can be missing.
1630 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1631 return 0;
1632
1633 return FullArgCount;
1634}
1635
1636static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1637 Intrinsic::ID IID = F->getIntrinsicID();
1638
1639 unsigned FullArgCount = getFullArgCountForDefaultArgUpgrade(F, IID);
1640 if (FullArgCount == 0)
1641 return false;
1642
1643 rename(F);
1644 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1645 assert(NewFn->arg_size() == FullArgCount &&
1646 "total number of default args does not match intrinsic signature");
1647 return true;
1648}
1649
1650static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1651 bool CanUpgradeDebugIntrinsicsToRecords) {
1652 assert(F && "Illegal to upgrade a non-existent Function.");
1653
1654 StringRef Name = F->getName();
1655
1656 // Quickly eliminate it, if it's not a candidate.
1657 if (!Name.consume_front("llvm.") || Name.empty())
1658 return false;
1659
1660 switch (Name[0]) {
1661 default: break;
1662 case 'a': {
1663 bool IsArm = Name.consume_front("arm.");
1664 if (IsArm || Name.consume_front("aarch64.")) {
1665 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1666 return true;
1667 break;
1668 }
1669
1670 if (Name.consume_front("amdgcn.")) {
1671 if (Name == "alignbit") {
1672 // Target specific intrinsic became redundant
1673 NewFn = Intrinsic::getOrInsertDeclaration(
1674 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1675 return true;
1676 }
1677
1678 if (Name.consume_front("atomic.")) {
1679 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1680 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1681 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1682 // and usub_sat so there's no new declaration.
1683 NewFn = nullptr;
1684 return true;
1685 }
1686 break; // No other 'amdgcn.atomic.*'
1687 }
1688
1689 if (Name.starts_with("addrspacecast.nonnull")) {
1690 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1691 // so there's no new declaration.
1692 NewFn = nullptr;
1693 return true;
1694 }
1695
1696 switch (F->getIntrinsicID()) {
1697 default:
1698 break;
1699 // Legacy wmma iu intrinsics without the optional clamp operand.
1700 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1701 if (F->arg_size() == 7) {
1702 NewFn = nullptr;
1703 return true;
1704 }
1705 break;
1706 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1707 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1708 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1709 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1710 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1711 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1712 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1713 if (F->arg_size() == 8) {
1714 NewFn = nullptr;
1715 return true;
1716 }
1717 break;
1718 }
1719
1720 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1721 Name.consume_front("flat.atomic.")) {
1722 if (Name.starts_with("fadd") ||
1723 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1724 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1725 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1726 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1727 // declaration.
1728 NewFn = nullptr;
1729 return true;
1730 }
1731 }
1732
1733 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1734 NewFn = nullptr;
1735 return true;
1736 }
1737
1738 if (Name.starts_with("ldexp.")) {
1739 // Target specific intrinsic became redundant
1740 NewFn = Intrinsic::getOrInsertDeclaration(
1741 F->getParent(), Intrinsic::ldexp,
1742 {F->getReturnType(), F->getArg(1)->getType()});
1743 return true;
1744 }
1745 break; // No other 'amdgcn.*'
1746 }
1747
1748 break;
1749 }
1750 case 'c': {
1751 if (F->arg_size() == 1) {
1752 if (Name.consume_front("convert.")) {
1753 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1754 NewFn = nullptr;
1755 return true;
1756 }
1757 }
1758
1759 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1760 .StartsWith("ctlz.", Intrinsic::ctlz)
1761 .StartsWith("cttz.", Intrinsic::cttz)
1762 .Default(Intrinsic::not_intrinsic);
1763 if (ID != Intrinsic::not_intrinsic) {
1764 rename(F);
1765 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1766 F->arg_begin()->getType());
1767 return true;
1768 }
1769 }
1770
1771 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1772 if (Name == "coro.end" &&
1773 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1774 CoroEndID = Intrinsic::coro_end;
1775 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1776 CoroEndID = Intrinsic::coro_end_async;
1777
1778 if (CoroEndID != Intrinsic::not_intrinsic) {
1779 rename(F);
1780 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1781 return true;
1782 }
1783
1784 break;
1785 }
1786 case 'd':
1787 if (Name.consume_front("dbg.")) {
1788 // Mark debug intrinsics for upgrade to new debug format.
1789 if (CanUpgradeDebugIntrinsicsToRecords) {
1790 if (Name == "addr" || Name == "value" || Name == "assign" ||
1791 Name == "declare" || Name == "label") {
1792 // There's no function to replace these with.
1793 NewFn = nullptr;
1794 // But we do want these to get upgraded.
1795 return true;
1796 }
1797 }
1798 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1799 // converted to DbgVariableRecords later.
1800 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1801 rename(F);
1802 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1803 Intrinsic::dbg_value);
1804 return true;
1805 }
1806 break; // No other 'dbg.*'.
1807 }
1808 break;
1809 case 'e':
1810 if (Name.consume_front("experimental.vector.")) {
1811 Intrinsic::ID ID =
1812 StringSwitch<Intrinsic::ID>(Name)
1813 // Skip over extract.last.active, otherwise it will be 'upgraded'
1814 // to a regular vector extract which is a different operation.
1815 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1816 .StartsWith("extract.", Intrinsic::vector_extract)
1817 .StartsWith("insert.", Intrinsic::vector_insert)
1818 .StartsWith("reverse.", Intrinsic::vector_reverse)
1819 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1820 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1821 .StartsWith("partial.reduce.add",
1822 Intrinsic::vector_partial_reduce_add)
1823 .Default(Intrinsic::not_intrinsic);
1824 if (ID != Intrinsic::not_intrinsic) {
1825 const auto *FT = F->getFunctionType();
1826 SmallVector<Type *, 2> Tys;
1827 if (ID == Intrinsic::vector_extract ||
1828 ID == Intrinsic::vector_interleave2)
1829 // Extracting overloads the return type.
1830 Tys.push_back(FT->getReturnType());
1831 if (ID != Intrinsic::vector_interleave2)
1832 Tys.push_back(FT->getParamType(0));
1833 if (ID == Intrinsic::vector_insert ||
1834 ID == Intrinsic::vector_partial_reduce_add)
1835 // Inserting overloads the inserted type.
1836 Tys.push_back(FT->getParamType(1));
1837 rename(F);
1838 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1839 return true;
1840 }
1841
1842 if (Name.consume_front("reduce.")) {
1843 SmallVector<StringRef, 2> Groups;
1844 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1845 if (R.match(Name, &Groups))
1846 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1847 .Case("add", Intrinsic::vector_reduce_add)
1848 .Case("mul", Intrinsic::vector_reduce_mul)
1849 .Case("and", Intrinsic::vector_reduce_and)
1850 .Case("or", Intrinsic::vector_reduce_or)
1851 .Case("xor", Intrinsic::vector_reduce_xor)
1852 .Case("smax", Intrinsic::vector_reduce_smax)
1853 .Case("smin", Intrinsic::vector_reduce_smin)
1854 .Case("umax", Intrinsic::vector_reduce_umax)
1855 .Case("umin", Intrinsic::vector_reduce_umin)
1856 .Case("fmax", Intrinsic::vector_reduce_fmax)
1857 .Case("fmin", Intrinsic::vector_reduce_fmin)
1858 .Default(Intrinsic::not_intrinsic);
1859
1860 bool V2 = false;
1861 if (ID == Intrinsic::not_intrinsic) {
1862 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1863 Groups.clear();
1864 V2 = true;
1865 if (R2.match(Name, &Groups))
1866 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1867 .Case("fadd", Intrinsic::vector_reduce_fadd)
1868 .Case("fmul", Intrinsic::vector_reduce_fmul)
1869 .Default(Intrinsic::not_intrinsic);
1870 }
1871 if (ID != Intrinsic::not_intrinsic) {
1872 rename(F);
1873 auto Args = F->getFunctionType()->params();
1874 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1875 {Args[V2 ? 1 : 0]});
1876 return true;
1877 }
1878 break; // No other 'expermental.vector.reduce.*'.
1879 }
1880
1881 if (Name.consume_front("splice"))
1882 return true;
1883 break; // No other 'experimental.vector.*'.
1884 }
1885 if (Name.consume_front("experimental.stepvector.")) {
1886 Intrinsic::ID ID = Intrinsic::stepvector;
1887 rename(F);
1888 NewFn = Intrinsic::getOrInsertDeclaration(
1889 F->getParent(), ID, F->getFunctionType()->getReturnType());
1890 return true;
1891 }
1892 break; // No other 'e*'.
1893 case 'f':
1894 if (Name.starts_with("flt.rounds")) {
1895 rename(F);
1896 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1897 Intrinsic::get_rounding);
1898 return true;
1899 }
1900 break;
1901 case 'i':
1902 if (Name.starts_with("invariant.group.barrier")) {
1903 // Rename invariant.group.barrier to launder.invariant.group
1904 auto Args = F->getFunctionType()->params();
1905 Type* ObjectPtr[1] = {Args[0]};
1906 rename(F);
1907 NewFn = Intrinsic::getOrInsertDeclaration(
1908 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1909 return true;
1910 }
1911 break;
1912 case 'l': {
1913 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1914 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1915 if (IsLifetimeStart || IsLifetimeEnd) {
1916 if (F->arg_size() == 2) {
1917 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1918 : Intrinsic::lifetime_end;
1919 rename(F);
1920 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1921 // arguments. Use the Ptr argument to create new declaration.
1922 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1923 F->getArg(1)->getType());
1924 return true;
1925 } else if (F->arg_size() == 1 && Name == ".i64") {
1926 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1927 // Autoupgrade prior to
1928 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1929 // intrinsic with no expected calls. To allow auto-upgrade process to
1930 // delete such invalid intrinsic declaration, set NewFn = nullptr
1931 // and return true here. If there are actual calls to this intrinsic
1932 // (which is not expected), they will be deleted in
1933 // UpgradeIntrinsicCall.
1934 NewFn = nullptr;
1935 return true;
1936 }
1937 }
1938 break;
1939 }
1940 case 'm': {
1941 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1942 // alignment parameter to embedding the alignment as an attribute of
1943 // the pointer args.
1944 if (unsigned ID = StringSwitch<unsigned>(Name)
1945 .StartsWith("memcpy.", Intrinsic::memcpy)
1946 .StartsWith("memmove.", Intrinsic::memmove)
1947 .Default(0)) {
1948 if (F->arg_size() == 5) {
1949 rename(F);
1950 // Get the types of dest, src, and len
1951 ArrayRef<Type *> ParamTypes =
1952 F->getFunctionType()->params().slice(0, 3);
1953 NewFn =
1954 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1955 return true;
1956 }
1957 }
1958 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1959 rename(F);
1960 // Get the types of dest, and len
1961 const auto *FT = F->getFunctionType();
1962 Type *ParamTypes[2] = {
1963 FT->getParamType(0), // Dest
1964 FT->getParamType(2) // len
1965 };
1966 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1967 Intrinsic::memset, ParamTypes);
1968 return true;
1969 }
1970
1971 unsigned MaskedID =
1972 StringSwitch<unsigned>(Name)
1973 .StartsWith("masked.load", Intrinsic::masked_load)
1974 .StartsWith("masked.gather", Intrinsic::masked_gather)
1975 .StartsWith("masked.store", Intrinsic::masked_store)
1976 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1977 .Default(0);
1978 if (MaskedID && F->arg_size() == 4) {
1979 rename(F);
1980 if (MaskedID == Intrinsic::masked_load ||
1981 MaskedID == Intrinsic::masked_gather) {
1982 NewFn = Intrinsic::getOrInsertDeclaration(
1983 F->getParent(), MaskedID,
1984 {F->getReturnType(), F->getArg(0)->getType()});
1985 return true;
1986 }
1987 NewFn = Intrinsic::getOrInsertDeclaration(
1988 F->getParent(), MaskedID,
1989 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1990 return true;
1991 }
1992 break;
1993 }
1994 case 'n': {
1995 if (Name.consume_front("nvvm.")) {
1996 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1997 if (F->arg_size() == 1) {
1998 Intrinsic::ID IID =
1999 StringSwitch<Intrinsic::ID>(Name)
2000 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2001 .Case("clz.i", Intrinsic::ctlz)
2002 .Case("popc.i", Intrinsic::ctpop)
2003 .Default(Intrinsic::not_intrinsic);
2004 if (IID != Intrinsic::not_intrinsic) {
2005 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2006 {F->getReturnType()});
2007 return true;
2008 }
2009 } else if (F->arg_size() == 2) {
2010 Intrinsic::ID IID =
2011 StringSwitch<Intrinsic::ID>(Name)
2012 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2013 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2014 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2015 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2016 .Default(Intrinsic::not_intrinsic);
2017 if (IID != Intrinsic::not_intrinsic) {
2018 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2019 {F->getReturnType()});
2020 return true;
2021 }
2022 }
2023
2024 // Check for nvvm intrinsics that need a return type adjustment.
2025 {
2026 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2027 if (IID != Intrinsic::not_intrinsic &&
2028 isLegacyNVPTXBF16IntSignature(F, IID)) {
2029 NewFn = nullptr;
2030 return true;
2031 }
2032 }
2033
2034 // Upgrade Distributed Shared Memory Intrinsics
2035 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2036 if (IID != Intrinsic::not_intrinsic) {
2037 rename(F);
2038 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2039 return true;
2040 }
2041
2042 // Upgrade TMA reduction intrinsics
2043 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2044 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2045 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2046 if (IID != Intrinsic::not_intrinsic) {
2047 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2048 return true;
2049 }
2050
2051 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2052 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2053 if (IID != Intrinsic::not_intrinsic) {
2054 rename(F);
2055 NewFn = Intrinsic::getOrInsertDeclaration(
2056 F->getParent(), IID, F->getReturnType(),
2057 F->getFunctionType()->params());
2058 return true;
2059 }
2060
2061 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2062 // tcgen05.alloc shared variants to anyptr intrinsics.
2063 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2064 if (IID != Intrinsic::not_intrinsic) {
2065 rename(F);
2066 if (Intrinsic::isOverloaded(IID))
2067 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2068 {F->getArg(0)->getType()});
2069 else
2070 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2071 return true;
2072 }
2073
2074 // Upgrade TMA copy G2S CTA intrinsics.
2075 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2076 if (IID != Intrinsic::not_intrinsic) {
2077 rename(F);
2078 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2079 return true;
2080 }
2081
2082 // Upgrade TMA copy G2S (cluster) intrinsics.
2083 SmallVector<Type *, 1> OvlTys;
2084 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2085 if (IID != Intrinsic::not_intrinsic) {
2086 rename(F);
2087 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2088 return true;
2089 }
2090
2091 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
2092 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
2093 if (IID != Intrinsic::not_intrinsic) {
2094 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2095 return NewFn != F;
2096 }
2097
2098 // Upgrade mbarrier.init intrinsics missing the layout operand.
2099 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2100 if (IID != Intrinsic::not_intrinsic) {
2101 rename(F);
2102 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2103 F->getArg(0)->getType());
2104 return true;
2105 }
2106
2107 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2108 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2109 //
2110 // TODO: We could add lohi.i2d.
2111 bool Expand = false;
2112 if (Name.consume_front("abs."))
2113 // nvvm.abs.{i,ii}
2114 Expand =
2115 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2116 else if (Name.consume_front("fabs."))
2117 // nvvm.fabs.{f,ftz.f,d}
2118 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2119 else if (Name.consume_front("add."))
2120 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2121 Expand = getNVVMFAddUpgrade(Name).has_value();
2122 else if (Name.consume_front("ex2.approx."))
2123 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2124 Expand =
2125 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2126 else if (Name.consume_front("atomic.load."))
2127 // nvvm.atomic.load.add.{f32,f64}.p
2128 // nvvm.atomic.load.{inc,dec}.32.p
2129 Expand = StringSwitch<bool>(Name)
2130 .StartsWith("add.f32.p", true)
2131 .StartsWith("add.f64.p", true)
2132 .StartsWith("inc.32.p", true)
2133 .StartsWith("dec.32.p", true)
2134 .Default(false);
2135 else if (Name.consume_front("atomic."))
2136 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2137 // nvvm.atomic.cas.gen.i.{cta,sys}
2138 Expand = StringSwitch<bool>(Name)
2139 .StartsWith("add.gen.", true)
2140 .StartsWith("exch.gen.", true)
2141 .StartsWith("max.gen.", true)
2142 .StartsWith("min.gen.", true)
2143 .StartsWith("inc.gen.", true)
2144 .StartsWith("dec.gen.", true)
2145 .StartsWith("and.gen.", true)
2146 .StartsWith("or.gen.", true)
2147 .StartsWith("xor.gen.", true)
2148 .StartsWith("cas.gen.", true)
2149 .Default(false);
2150 else if (Name.consume_front("bitcast."))
2151 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2152 Expand =
2153 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2154 else if (Name.consume_front("rotate."))
2155 // nvvm.rotate.{b32,b64,right.b64}
2156 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2157 else if (Name.consume_front("ptr.gen.to."))
2158 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2159 Expand = consumeNVVMPtrAddrSpace(Name);
2160 else if (Name.consume_front("ptr."))
2161 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2162 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2163 else if (Name.consume_front("ldg.global."))
2164 // nvvm.ldg.global.{i,p,f}
2165 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2166 Name.starts_with("p."));
2167 else
2168 Expand = StringSwitch<bool>(Name)
2169 .Case("barrier0", true)
2170 .Case("barrier.n", true)
2171 .Case("barrier.sync.cnt", true)
2172 .Case("barrier.sync", true)
2173 .Case("barrier", true)
2174 .Case("bar.sync", true)
2175 .Case("barrier0.popc", true)
2176 .Case("barrier0.and", true)
2177 .Case("barrier0.or", true)
2178 .Case("clz.ll", true)
2179 .Case("popc.ll", true)
2180 .Case("h2f", true)
2181 .Case("swap.lo.hi.b64", true)
2182 .Case("tanh.approx.f32", true)
2183 .Default(false);
2184
2185 if (Expand) {
2186 NewFn = nullptr;
2187 return true;
2188 }
2189 break; // No other 'nvvm.*'.
2190 }
2191 break;
2192 }
2193 case 'o':
2194 if (Name.starts_with("objectsize.")) {
2195 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2196 if (F->arg_size() == 2 || F->arg_size() == 3) {
2197 rename(F);
2198 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2199 Intrinsic::objectsize, Tys);
2200 return true;
2201 }
2202 }
2203 break;
2204
2205 case 'p':
2206 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2207 rename(F);
2208 NewFn = Intrinsic::getOrInsertDeclaration(
2209 F->getParent(), Intrinsic::ptr_annotation,
2210 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2211 return true;
2212 }
2213 break;
2214
2215 case 'r': {
2216 if (Name.consume_front("riscv.")) {
2217 Intrinsic::ID ID;
2218 ID = StringSwitch<Intrinsic::ID>(Name)
2219 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2220 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2221 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2222 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2223 .Default(Intrinsic::not_intrinsic);
2224 if (ID != Intrinsic::not_intrinsic) {
2225 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2226 rename(F);
2227 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2228 return true;
2229 }
2230 break; // No other applicable upgrades.
2231 }
2232
2233 ID = StringSwitch<Intrinsic::ID>(Name)
2234 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2235 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2236 .Default(Intrinsic::not_intrinsic);
2237 if (ID != Intrinsic::not_intrinsic) {
2238 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2239 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2240 rename(F);
2241 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2242 return true;
2243 }
2244 break; // No other applicable upgrades.
2245 }
2246
2247 ID = StringSwitch<Intrinsic::ID>(Name)
2248 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2249 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2250 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2251 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2252 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2253 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2254 .Default(Intrinsic::not_intrinsic);
2255 if (ID != Intrinsic::not_intrinsic) {
2256 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2257 rename(F);
2258 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2259 return true;
2260 }
2261 break; // No other applicable upgrades.
2262 }
2263
2264 // Replace llvm.riscv.clmul with llvm.clmul.
2265 if (Name == "clmul.i32" || Name == "clmul.i64") {
2266 NewFn = Intrinsic::getOrInsertDeclaration(
2267 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2268 return true;
2269 }
2270
2271 break; // No other 'riscv.*' intrinsics
2272 }
2273 } break;
2274
2275 case 's':
2276 if (Name == "stackprotectorcheck") {
2277 NewFn = nullptr;
2278 return true;
2279 }
2280 break;
2281
2282 case 't':
2283 if (Name == "thread.pointer") {
2284 NewFn = Intrinsic::getOrInsertDeclaration(
2285 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2286 return true;
2287 }
2288 break;
2289
2290 case 'v': {
2291 if (Name == "var.annotation" && F->arg_size() == 4) {
2292 rename(F);
2293 NewFn = Intrinsic::getOrInsertDeclaration(
2294 F->getParent(), Intrinsic::var_annotation,
2295 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2296 return true;
2297 }
2298 if (Name.consume_front("vector.splice")) {
2299 if (Name.starts_with(".left") || Name.starts_with(".right"))
2300 break;
2301 return true;
2302 }
2303 if (shouldUpgradeVPIntrinsic(Name))
2304 return true;
2305 break;
2306 }
2307
2308 case 'w':
2309 if (Name.consume_front("wasm.")) {
2310 Intrinsic::ID ID =
2311 StringSwitch<Intrinsic::ID>(Name)
2312 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2313 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2314 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2315 .Default(Intrinsic::not_intrinsic);
2316 if (ID != Intrinsic::not_intrinsic) {
2317 rename(F);
2318 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2319 F->getReturnType());
2320 return true;
2321 }
2322
2323 if (Name.consume_front("dot.i8x16.i7x16.")) {
2324 ID = StringSwitch<Intrinsic::ID>(Name)
2325 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2326 .Case("add.signed",
2327 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2328 .Default(Intrinsic::not_intrinsic);
2329 if (ID != Intrinsic::not_intrinsic) {
2330 rename(F);
2331 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2332 return true;
2333 }
2334 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2335 }
2336 break; // No other 'wasm.*'.
2337 }
2338 break;
2339
2340 case 'x':
2341 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2342 return true;
2343 }
2344
2345 auto *ST = dyn_cast<StructType>(F->getReturnType());
2346 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2347 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2348 // Replace return type with literal non-packed struct. Only do this for
2349 // intrinsics declared to return a struct, not for intrinsics with
2350 // overloaded return type, in which case the exact struct type will be
2351 // mangled into the name.
2352 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2353 FunctionType *FT = F->getFunctionType();
2354 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2355 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2356 std::string Name = F->getName().str();
2357 rename(F);
2358 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2359 Name, F->getParent());
2360
2361 // The new function may also need remangling.
2362 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2363 NewFn = *Result;
2364 return true;
2365 }
2366 }
2367
2368 // Remangle our intrinsic since we upgrade the mangling
2369 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2370 if (Result != std::nullopt) {
2371 NewFn = *Result;
2372 return true;
2373 }
2374
2375 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2376 return true;
2377
2378 // This may not belong here. This function is effectively being overloaded
2379 // to both detect an intrinsic which needs upgrading, and to provide the
2380 // upgraded form of the intrinsic. We should perhaps have two separate
2381 // functions for this.
2382
2383 return false;
2384}
2385
2386bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2387 bool CanUpgradeDebugIntrinsicsToRecords) {
2388 NewFn = nullptr;
2389 bool Upgraded =
2390 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2391
2392 // Upgrade intrinsic attributes. This does not change the function.
2393 if (NewFn)
2394 F = NewFn;
2395 if (Intrinsic::ID id = F->getIntrinsicID()) {
2396 // Only do this if the intrinsic signature is valid.
2397 SmallVector<Type *> OverloadTys;
2398 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2399 F->setAttributes(
2400 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2401 }
2402 return Upgraded;
2403}
2404
2405GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2406 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2407 GV->getName() == "llvm.global_dtors")) ||
2408 !GV->hasInitializer())
2409 return nullptr;
2410 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2411 if (!ATy)
2412 return nullptr;
2413 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2414 if (!STy || STy->getNumElements() != 2)
2415 return nullptr;
2416
2417 LLVMContext &C = GV->getContext();
2418 IRBuilder<> IRB(C);
2419 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2420 IRB.getPtrTy());
2421 Constant *Init = GV->getInitializer();
2422 unsigned N = Init->getNumOperands();
2423 std::vector<Constant *> NewCtors(N);
2424 for (unsigned i = 0; i != N; ++i) {
2425 auto Ctor = cast<Constant>(Init->getOperand(i));
2426 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2427 Ctor->getAggregateElement(1),
2428 ConstantPointerNull::get(IRB.getPtrTy()));
2429 }
2430 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2431
2432 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2433 NewInit, GV->getName());
2434}
2435
2436// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2437// to byte shuffles.
2438static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2439 unsigned Shift) {
2440 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2441 unsigned NumElts = ResultTy->getNumElements() * 8;
2442
2443 // Bitcast from a 64-bit element type to a byte element type.
2444 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2445 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2446
2447 // We'll be shuffling in zeroes.
2448 Value *Res = Constant::getNullValue(VecTy);
2449
2450 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2451 // we'll just return the zero vector.
2452 if (Shift < 16) {
2453 int Idxs[64];
2454 // 256/512-bit version is split into 2/4 16-byte lanes.
2455 for (unsigned l = 0; l != NumElts; l += 16)
2456 for (unsigned i = 0; i != 16; ++i) {
2457 unsigned Idx = NumElts + i - Shift;
2458 if (Idx < NumElts)
2459 Idx -= NumElts - 16; // end of lane, switch operand.
2460 Idxs[l + i] = Idx + l;
2461 }
2462
2463 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2464 }
2465
2466 // Bitcast back to a 64-bit element type.
2467 return Builder.CreateBitCast(Res, ResultTy, "cast");
2468}
2469
2470// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2471// to byte shuffles.
2472static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2473 unsigned Shift) {
2474 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2475 unsigned NumElts = ResultTy->getNumElements() * 8;
2476
2477 // Bitcast from a 64-bit element type to a byte element type.
2478 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2479 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2480
2481 // We'll be shuffling in zeroes.
2482 Value *Res = Constant::getNullValue(VecTy);
2483
2484 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2485 // we'll just return the zero vector.
2486 if (Shift < 16) {
2487 int Idxs[64];
2488 // 256/512-bit version is split into 2/4 16-byte lanes.
2489 for (unsigned l = 0; l != NumElts; l += 16)
2490 for (unsigned i = 0; i != 16; ++i) {
2491 unsigned Idx = i + Shift;
2492 if (Idx >= 16)
2493 Idx += NumElts - 16; // end of lane, switch operand.
2494 Idxs[l + i] = Idx + l;
2495 }
2496
2497 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2498 }
2499
2500 // Bitcast back to a 64-bit element type.
2501 return Builder.CreateBitCast(Res, ResultTy, "cast");
2502}
2503
2504static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2505 unsigned NumElts) {
2506 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2507 llvm::VectorType *MaskTy = FixedVectorType::get(
2508 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2509 Mask = Builder.CreateBitCast(Mask, MaskTy);
2510
2511 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2512 // i8 and we need to extract down to the right number of elements.
2513 if (NumElts <= 4) {
2514 int Indices[4];
2515 for (unsigned i = 0; i != NumElts; ++i)
2516 Indices[i] = i;
2517 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2518 "extract");
2519 }
2520
2521 return Mask;
2522}
2523
2524static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2525 Value *Op1) {
2526 // If the mask is all ones just emit the first operation.
2527 if (const auto *C = dyn_cast<Constant>(Mask))
2528 if (C->isAllOnesValue())
2529 return Op0;
2530
2531 Mask = getX86MaskVec(Builder, Mask,
2532 cast<FixedVectorType>(Op0->getType())->getNumElements());
2533 return Builder.CreateSelect(Mask, Op0, Op1);
2534}
2535
2536static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2537 Value *Op1) {
2538 // If the mask is all ones just emit the first operation.
2539 if (const auto *C = dyn_cast<Constant>(Mask))
2540 if (C->isAllOnesValue())
2541 return Op0;
2542
2543 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2544 Mask->getType()->getIntegerBitWidth());
2545 Mask = Builder.CreateBitCast(Mask, MaskTy);
2546 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2547 return Builder.CreateSelect(Mask, Op0, Op1);
2548}
2549
2550// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2551// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2552// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2553static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2554 Value *Op1, Value *Shift,
2555 Value *Passthru, Value *Mask,
2556 bool IsVALIGN) {
2557 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2558
2559 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2560 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2561 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2562 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2563
2564 // Mask the immediate for VALIGN.
2565 if (IsVALIGN)
2566 ShiftVal &= (NumElts - 1);
2567
2568 // If palignr is shifting the pair of vectors more than the size of two
2569 // lanes, emit zero.
2570 if (ShiftVal >= 32)
2571 return llvm::Constant::getNullValue(Op0->getType());
2572
2573 // If palignr is shifting the pair of input vectors more than one lane,
2574 // but less than two lanes, convert to shifting in zeroes.
2575 if (ShiftVal > 16) {
2576 ShiftVal -= 16;
2577 Op1 = Op0;
2578 Op0 = llvm::Constant::getNullValue(Op0->getType());
2579 }
2580
2581 int Indices[64];
2582 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2583 for (unsigned l = 0; l < NumElts; l += 16) {
2584 for (unsigned i = 0; i != 16; ++i) {
2585 unsigned Idx = ShiftVal + i;
2586 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2587 Idx += NumElts - 16; // End of lane, switch operand.
2588 Indices[l + i] = Idx + l;
2589 }
2590 }
2591
2592 Value *Align = Builder.CreateShuffleVector(
2593 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2594
2595 return emitX86Select(Builder, Mask, Align, Passthru);
2596}
2597
2598static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2599 bool ZeroMask, bool IndexForm) {
2600 Type *Ty = CI.getType();
2601 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2602 unsigned EltWidth = Ty->getScalarSizeInBits();
2603 bool IsFloat = Ty->isFPOrFPVectorTy();
2604 Intrinsic::ID IID;
2605 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2606 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2607 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2608 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2609 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2610 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2611 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2612 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2613 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2614 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2615 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2616 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2617 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2618 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2619 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2620 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2621 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2622 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2623 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2624 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2625 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2626 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2627 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2628 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2629 else if (VecWidth == 128 && EltWidth == 16)
2630 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2631 else if (VecWidth == 256 && EltWidth == 16)
2632 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2633 else if (VecWidth == 512 && EltWidth == 16)
2634 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2635 else if (VecWidth == 128 && EltWidth == 8)
2636 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2637 else if (VecWidth == 256 && EltWidth == 8)
2638 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2639 else if (VecWidth == 512 && EltWidth == 8)
2640 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2641 else
2642 llvm_unreachable("Unexpected intrinsic");
2643
2644 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2645 CI.getArgOperand(2) };
2646
2647 // If this isn't index form we need to swap operand 0 and 1.
2648 if (!IndexForm)
2649 std::swap(Args[0], Args[1]);
2650
2651 Value *V = Builder.CreateIntrinsic(IID, Args);
2652 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2653 : Builder.CreateBitCast(CI.getArgOperand(1),
2654 Ty);
2655 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2656}
2657
2658static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2659 Intrinsic::ID IID) {
2660 Type *Ty = CI.getType();
2661 Value *Op0 = CI.getOperand(0);
2662 Value *Op1 = CI.getOperand(1);
2663 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2664
2665 if (CI.arg_size() == 4) { // For masked intrinsics.
2666 Value *VecSrc = CI.getOperand(2);
2667 Value *Mask = CI.getOperand(3);
2668 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2669 }
2670 return Res;
2671}
2672
2673static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2674 bool IsRotateRight) {
2675 Type *Ty = CI.getType();
2676 Value *Src = CI.getArgOperand(0);
2677 Value *Amt = CI.getArgOperand(1);
2678
2679 // Amount may be scalar immediate, in which case create a splat vector.
2680 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2681 // we only care about the lowest log2 bits anyway.
2682 if (Amt->getType() != Ty) {
2683 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2684 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2685 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2686 }
2687
2688 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2689 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2690
2691 if (CI.arg_size() == 4) { // For masked intrinsics.
2692 Value *VecSrc = CI.getOperand(2);
2693 Value *Mask = CI.getOperand(3);
2694 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2695 }
2696 return Res;
2697}
2698
2699static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2700 bool IsSigned) {
2701 Type *Ty = CI.getType();
2702 Value *LHS = CI.getArgOperand(0);
2703 Value *RHS = CI.getArgOperand(1);
2704
2705 CmpInst::Predicate Pred;
2706 switch (Imm) {
2707 case 0x0:
2708 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2709 break;
2710 case 0x1:
2711 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2712 break;
2713 case 0x2:
2714 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2715 break;
2716 case 0x3:
2717 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2718 break;
2719 case 0x4:
2720 Pred = ICmpInst::ICMP_EQ;
2721 break;
2722 case 0x5:
2723 Pred = ICmpInst::ICMP_NE;
2724 break;
2725 case 0x6:
2726 return Constant::getNullValue(Ty); // FALSE
2727 case 0x7:
2728 return Constant::getAllOnesValue(Ty); // TRUE
2729 default:
2730 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2731 }
2732
2733 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2734 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2735 return Ext;
2736}
2737
2738static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2739 bool IsShiftRight, bool ZeroMask) {
2740 Type *Ty = CI.getType();
2741 Value *Op0 = CI.getArgOperand(0);
2742 Value *Op1 = CI.getArgOperand(1);
2743 Value *Amt = CI.getArgOperand(2);
2744
2745 if (IsShiftRight)
2746 std::swap(Op0, Op1);
2747
2748 // Amount may be scalar immediate, in which case create a splat vector.
2749 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2750 // we only care about the lowest log2 bits anyway.
2751 if (Amt->getType() != Ty) {
2752 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2753 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2754 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2755 }
2756
2757 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2758 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2759
2760 unsigned NumArgs = CI.arg_size();
2761 if (NumArgs >= 4) { // For masked intrinsics.
2762 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2763 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2764 CI.getArgOperand(0);
2765 Value *Mask = CI.getOperand(NumArgs - 1);
2766 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2767 }
2768 return Res;
2769}
2770
2771static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2772 Value *Mask, bool Aligned) {
2773 const Align Alignment =
2774 Aligned
2775 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2776 : Align(1);
2777
2778 // If the mask is all ones just emit a regular store.
2779 if (const auto *C = dyn_cast<Constant>(Mask))
2780 if (C->isAllOnesValue())
2781 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2782
2783 // Convert the mask from an integer type to a vector of i1.
2784 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2785 Mask = getX86MaskVec(Builder, Mask, NumElts);
2786 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2787}
2788
2789static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2790 Value *Passthru, Value *Mask, bool Aligned) {
2791 Type *ValTy = Passthru->getType();
2792 const Align Alignment =
2793 Aligned
2794 ? Align(
2795 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2796 8)
2797 : Align(1);
2798
2799 // If the mask is all ones just emit a regular store.
2800 if (const auto *C = dyn_cast<Constant>(Mask))
2801 if (C->isAllOnesValue())
2802 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2803
2804 // Convert the mask from an integer type to a vector of i1.
2805 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2806 Mask = getX86MaskVec(Builder, Mask, NumElts);
2807 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2808}
2809
2810static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2811 Type *Ty = CI.getType();
2812 Value *Op0 = CI.getArgOperand(0);
2813 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2814 {Op0, Builder.getInt1(false)});
2815 if (CI.arg_size() == 3)
2816 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2817 return Res;
2818}
2819
2820static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2821 Type *Ty = CI.getType();
2822
2823 // Arguments have a vXi32 type so cast to vXi64.
2824 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2825 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2826
2827 if (IsSigned) {
2828 // Shift left then arithmetic shift right.
2829 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2830 LHS = Builder.CreateShl(LHS, ShiftAmt);
2831 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2832 RHS = Builder.CreateShl(RHS, ShiftAmt);
2833 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2834 } else {
2835 // Clear the upper bits.
2836 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2837 LHS = Builder.CreateAnd(LHS, Mask);
2838 RHS = Builder.CreateAnd(RHS, Mask);
2839 }
2840
2841 Value *Res = Builder.CreateMul(LHS, RHS);
2842
2843 if (CI.arg_size() == 4)
2844 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2845
2846 return Res;
2847}
2848
2849// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2850static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2851 Value *Mask) {
2852 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2853 if (Mask) {
2854 const auto *C = dyn_cast<Constant>(Mask);
2855 if (!C || !C->isAllOnesValue())
2856 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2857 }
2858
2859 if (NumElts < 8) {
2860 int Indices[8];
2861 for (unsigned i = 0; i != NumElts; ++i)
2862 Indices[i] = i;
2863 for (unsigned i = NumElts; i != 8; ++i)
2864 Indices[i] = NumElts + i % NumElts;
2865 Vec = Builder.CreateShuffleVector(Vec,
2866 Constant::getNullValue(Vec->getType()),
2867 Indices);
2868 }
2869 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2870}
2871
2872static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2873 unsigned CC, bool Signed) {
2874 Value *Op0 = CI.getArgOperand(0);
2875 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2876
2877 Value *Cmp;
2878 if (CC == 3) {
2879 Cmp = Constant::getNullValue(
2880 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2881 } else if (CC == 7) {
2882 Cmp = Constant::getAllOnesValue(
2883 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2884 } else {
2885 ICmpInst::Predicate Pred;
2886 switch (CC) {
2887 default: llvm_unreachable("Unknown condition code");
2888 case 0: Pred = ICmpInst::ICMP_EQ; break;
2889 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2890 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2891 case 4: Pred = ICmpInst::ICMP_NE; break;
2892 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2893 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2894 }
2895 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2896 }
2897
2898 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2899
2900 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2901}
2902
2903// Replace a masked intrinsic with an older unmasked intrinsic.
2904static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2905 Intrinsic::ID IID) {
2906 Value *Rep =
2907 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2908 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2909}
2910
2911static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2912 Value* A = CI.getArgOperand(0);
2913 Value* B = CI.getArgOperand(1);
2914 Value* Src = CI.getArgOperand(2);
2915 Value* Mask = CI.getArgOperand(3);
2916
2917 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2918 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2919 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2920 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2921 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2922 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2923}
2924
2925static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2926 Value* Op = CI.getArgOperand(0);
2927 Type* ReturnOp = CI.getType();
2928 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2929 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2930 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2931}
2932
2933// Replace intrinsic with unmasked version and a select.
2934static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2935 CallBase &CI, Value *&Rep) {
2936 Name = Name.substr(12); // Remove avx512.mask.
2937
2938 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2939 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2940 Intrinsic::ID IID;
2941 if (Name.starts_with("max.p")) {
2942 if (VecWidth == 128 && EltWidth == 32)
2943 IID = Intrinsic::x86_sse_max_ps;
2944 else if (VecWidth == 128 && EltWidth == 64)
2945 IID = Intrinsic::x86_sse2_max_pd;
2946 else if (VecWidth == 256 && EltWidth == 32)
2947 IID = Intrinsic::x86_avx_max_ps_256;
2948 else if (VecWidth == 256 && EltWidth == 64)
2949 IID = Intrinsic::x86_avx_max_pd_256;
2950 else
2951 llvm_unreachable("Unexpected intrinsic");
2952 } else if (Name.starts_with("min.p")) {
2953 if (VecWidth == 128 && EltWidth == 32)
2954 IID = Intrinsic::x86_sse_min_ps;
2955 else if (VecWidth == 128 && EltWidth == 64)
2956 IID = Intrinsic::x86_sse2_min_pd;
2957 else if (VecWidth == 256 && EltWidth == 32)
2958 IID = Intrinsic::x86_avx_min_ps_256;
2959 else if (VecWidth == 256 && EltWidth == 64)
2960 IID = Intrinsic::x86_avx_min_pd_256;
2961 else
2962 llvm_unreachable("Unexpected intrinsic");
2963 } else if (Name.starts_with("pshuf.b.")) {
2964 if (VecWidth == 128)
2965 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2966 else if (VecWidth == 256)
2967 IID = Intrinsic::x86_avx2_pshuf_b;
2968 else if (VecWidth == 512)
2969 IID = Intrinsic::x86_avx512_pshuf_b_512;
2970 else
2971 llvm_unreachable("Unexpected intrinsic");
2972 } else if (Name.starts_with("pmul.hr.sw.")) {
2973 if (VecWidth == 128)
2974 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2975 else if (VecWidth == 256)
2976 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2977 else if (VecWidth == 512)
2978 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2979 else
2980 llvm_unreachable("Unexpected intrinsic");
2981 } else if (Name.starts_with("pmulh.w.")) {
2982 if (VecWidth == 128)
2983 IID = Intrinsic::x86_sse2_pmulh_w;
2984 else if (VecWidth == 256)
2985 IID = Intrinsic::x86_avx2_pmulh_w;
2986 else if (VecWidth == 512)
2987 IID = Intrinsic::x86_avx512_pmulh_w_512;
2988 else
2989 llvm_unreachable("Unexpected intrinsic");
2990 } else if (Name.starts_with("pmulhu.w.")) {
2991 if (VecWidth == 128)
2992 IID = Intrinsic::x86_sse2_pmulhu_w;
2993 else if (VecWidth == 256)
2994 IID = Intrinsic::x86_avx2_pmulhu_w;
2995 else if (VecWidth == 512)
2996 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2997 else
2998 llvm_unreachable("Unexpected intrinsic");
2999 } else if (Name.starts_with("pmaddw.d.")) {
3000 if (VecWidth == 128)
3001 IID = Intrinsic::x86_sse2_pmadd_wd;
3002 else if (VecWidth == 256)
3003 IID = Intrinsic::x86_avx2_pmadd_wd;
3004 else if (VecWidth == 512)
3005 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3006 else
3007 llvm_unreachable("Unexpected intrinsic");
3008 } else if (Name.starts_with("pmaddubs.w.")) {
3009 if (VecWidth == 128)
3010 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3011 else if (VecWidth == 256)
3012 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3013 else if (VecWidth == 512)
3014 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3015 else
3016 llvm_unreachable("Unexpected intrinsic");
3017 } else if (Name.starts_with("packsswb.")) {
3018 if (VecWidth == 128)
3019 IID = Intrinsic::x86_sse2_packsswb_128;
3020 else if (VecWidth == 256)
3021 IID = Intrinsic::x86_avx2_packsswb;
3022 else if (VecWidth == 512)
3023 IID = Intrinsic::x86_avx512_packsswb_512;
3024 else
3025 llvm_unreachable("Unexpected intrinsic");
3026 } else if (Name.starts_with("packssdw.")) {
3027 if (VecWidth == 128)
3028 IID = Intrinsic::x86_sse2_packssdw_128;
3029 else if (VecWidth == 256)
3030 IID = Intrinsic::x86_avx2_packssdw;
3031 else if (VecWidth == 512)
3032 IID = Intrinsic::x86_avx512_packssdw_512;
3033 else
3034 llvm_unreachable("Unexpected intrinsic");
3035 } else if (Name.starts_with("packuswb.")) {
3036 if (VecWidth == 128)
3037 IID = Intrinsic::x86_sse2_packuswb_128;
3038 else if (VecWidth == 256)
3039 IID = Intrinsic::x86_avx2_packuswb;
3040 else if (VecWidth == 512)
3041 IID = Intrinsic::x86_avx512_packuswb_512;
3042 else
3043 llvm_unreachable("Unexpected intrinsic");
3044 } else if (Name.starts_with("packusdw.")) {
3045 if (VecWidth == 128)
3046 IID = Intrinsic::x86_sse41_packusdw;
3047 else if (VecWidth == 256)
3048 IID = Intrinsic::x86_avx2_packusdw;
3049 else if (VecWidth == 512)
3050 IID = Intrinsic::x86_avx512_packusdw_512;
3051 else
3052 llvm_unreachable("Unexpected intrinsic");
3053 } else if (Name.starts_with("vpermilvar.")) {
3054 if (VecWidth == 128 && EltWidth == 32)
3055 IID = Intrinsic::x86_avx_vpermilvar_ps;
3056 else if (VecWidth == 128 && EltWidth == 64)
3057 IID = Intrinsic::x86_avx_vpermilvar_pd;
3058 else if (VecWidth == 256 && EltWidth == 32)
3059 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3060 else if (VecWidth == 256 && EltWidth == 64)
3061 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3062 else if (VecWidth == 512 && EltWidth == 32)
3063 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3064 else if (VecWidth == 512 && EltWidth == 64)
3065 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3066 else
3067 llvm_unreachable("Unexpected intrinsic");
3068 } else if (Name == "cvtpd2dq.256") {
3069 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3070 } else if (Name == "cvtpd2ps.256") {
3071 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3072 } else if (Name == "cvttpd2dq.256") {
3073 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3074 } else if (Name == "cvttps2dq.128") {
3075 IID = Intrinsic::x86_sse2_cvttps2dq;
3076 } else if (Name == "cvttps2dq.256") {
3077 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3078 } else if (Name.starts_with("permvar.")) {
3079 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3080 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3081 IID = Intrinsic::x86_avx2_permps;
3082 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3083 IID = Intrinsic::x86_avx2_permd;
3084 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3085 IID = Intrinsic::x86_avx512_permvar_df_256;
3086 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3087 IID = Intrinsic::x86_avx512_permvar_di_256;
3088 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3089 IID = Intrinsic::x86_avx512_permvar_sf_512;
3090 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3091 IID = Intrinsic::x86_avx512_permvar_si_512;
3092 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3093 IID = Intrinsic::x86_avx512_permvar_df_512;
3094 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3095 IID = Intrinsic::x86_avx512_permvar_di_512;
3096 else if (VecWidth == 128 && EltWidth == 16)
3097 IID = Intrinsic::x86_avx512_permvar_hi_128;
3098 else if (VecWidth == 256 && EltWidth == 16)
3099 IID = Intrinsic::x86_avx512_permvar_hi_256;
3100 else if (VecWidth == 512 && EltWidth == 16)
3101 IID = Intrinsic::x86_avx512_permvar_hi_512;
3102 else if (VecWidth == 128 && EltWidth == 8)
3103 IID = Intrinsic::x86_avx512_permvar_qi_128;
3104 else if (VecWidth == 256 && EltWidth == 8)
3105 IID = Intrinsic::x86_avx512_permvar_qi_256;
3106 else if (VecWidth == 512 && EltWidth == 8)
3107 IID = Intrinsic::x86_avx512_permvar_qi_512;
3108 else
3109 llvm_unreachable("Unexpected intrinsic");
3110 } else if (Name.starts_with("dbpsadbw.")) {
3111 if (VecWidth == 128)
3112 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3113 else if (VecWidth == 256)
3114 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3115 else if (VecWidth == 512)
3116 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3117 else
3118 llvm_unreachable("Unexpected intrinsic");
3119 } else if (Name.starts_with("pmultishift.qb.")) {
3120 if (VecWidth == 128)
3121 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3122 else if (VecWidth == 256)
3123 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3124 else if (VecWidth == 512)
3125 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3126 else
3127 llvm_unreachable("Unexpected intrinsic");
3128 } else if (Name.starts_with("conflict.")) {
3129 if (Name[9] == 'd' && VecWidth == 128)
3130 IID = Intrinsic::x86_avx512_conflict_d_128;
3131 else if (Name[9] == 'd' && VecWidth == 256)
3132 IID = Intrinsic::x86_avx512_conflict_d_256;
3133 else if (Name[9] == 'd' && VecWidth == 512)
3134 IID = Intrinsic::x86_avx512_conflict_d_512;
3135 else if (Name[9] == 'q' && VecWidth == 128)
3136 IID = Intrinsic::x86_avx512_conflict_q_128;
3137 else if (Name[9] == 'q' && VecWidth == 256)
3138 IID = Intrinsic::x86_avx512_conflict_q_256;
3139 else if (Name[9] == 'q' && VecWidth == 512)
3140 IID = Intrinsic::x86_avx512_conflict_q_512;
3141 else
3142 llvm_unreachable("Unexpected intrinsic");
3143 } else if (Name.starts_with("pavg.")) {
3144 if (Name[5] == 'b' && VecWidth == 128)
3145 IID = Intrinsic::x86_sse2_pavg_b;
3146 else if (Name[5] == 'b' && VecWidth == 256)
3147 IID = Intrinsic::x86_avx2_pavg_b;
3148 else if (Name[5] == 'b' && VecWidth == 512)
3149 IID = Intrinsic::x86_avx512_pavg_b_512;
3150 else if (Name[5] == 'w' && VecWidth == 128)
3151 IID = Intrinsic::x86_sse2_pavg_w;
3152 else if (Name[5] == 'w' && VecWidth == 256)
3153 IID = Intrinsic::x86_avx2_pavg_w;
3154 else if (Name[5] == 'w' && VecWidth == 512)
3155 IID = Intrinsic::x86_avx512_pavg_w_512;
3156 else
3157 llvm_unreachable("Unexpected intrinsic");
3158 } else
3159 return false;
3160
3161 SmallVector<Value *, 4> Args(CI.args());
3162 Args.pop_back();
3163 Args.pop_back();
3164 Rep = Builder.CreateIntrinsic(IID, Args);
3165 unsigned NumArgs = CI.arg_size();
3166 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3167 CI.getArgOperand(NumArgs - 2));
3168 return true;
3169}
3170
3171/// Upgrade comment in call to inline asm that represents an objc retain release
3172/// marker.
3173void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3174 size_t Pos;
3175 if (AsmStr->find("mov\tfp") == 0 &&
3176 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3177 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3178 AsmStr->replace(Pos, 1, ";");
3179 }
3180}
3181
3182static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3183 Function *F, IRBuilder<> &Builder) {
3184 Value *Rep = nullptr;
3185
3186 if (Name == "abs.i" || Name == "abs.ll") {
3187 Value *Arg = CI->getArgOperand(0);
3188 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3189 {Arg, Builder.getTrue()},
3190 /*FMFSource=*/nullptr, "abs");
3191 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3192 Type *Ty = (Name == "abs.bf16")
3193 ? Builder.getBFloatTy()
3194 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3195 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3196 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3197 Rep = Builder.CreateBitCast(Abs, CI->getType());
3198 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3199 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3200 : Intrinsic::nvvm_fabs;
3201 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3202 } else if (Name.consume_front("add.")) {
3203 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3204 auto FAdd = getNVVMFAddUpgrade(Name);
3205 assert(FAdd && "unsupported nvvm.add.* intrinsic");
3206 auto [IID, RoundingMode] = *FAdd;
3207 Value *A = CI->getArgOperand(0);
3208 Rep = Builder.CreateIntrinsic(
3209 A->getType(), IID,
3210 {A, CI->getArgOperand(1),
3211 Builder.getInt32(static_cast<int>(RoundingMode))});
3212 } else if (Name.consume_front("ex2.approx.")) {
3213 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3214 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3215 : Intrinsic::nvvm_ex2_approx;
3216 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3217 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3218 Name.starts_with("atomic.load.add.f64.p")) {
3219 Value *Ptr = CI->getArgOperand(0);
3220 Value *Val = CI->getArgOperand(1);
3221 Rep = Builder.CreateAtomicRMW(
3222 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3223 CI->getContext().getOrInsertSyncScopeID("device"));
3224 // The default scope for atomic.load.* intrinsics is device
3225 // (= gpu scope in ptx), but the default LLVM atomic scope is
3226 // "system"
3227 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3228 Name.starts_with("atomic.load.dec.32.p")) {
3229 Value *Ptr = CI->getArgOperand(0);
3230 Value *Val = CI->getArgOperand(1);
3231 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3232 : AtomicRMWInst::UDecWrap;
3233 Rep = Builder.CreateAtomicRMW(
3234 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3235 CI->getContext().getOrInsertSyncScopeID("device"));
3236 // See comment above.
3237 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3238 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3239 StringRef Op = Name.substr(StringRef("atomic.").size());
3240 Value *Ptr = CI->getArgOperand(0);
3241 Value *Val = CI->getArgOperand(1);
3242 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3243 Op.contains(".cta.") ? "block" : "");
3244 if (Op.starts_with("cas.")) {
3245 Value *New = CI->getArgOperand(2);
3246 Value *Pair = Builder.CreateAtomicCmpXchg(
3247 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3248 AtomicOrdering::Monotonic, SSID);
3249 Rep = Builder.CreateExtractValue(Pair, 0);
3250 } else {
3251 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3252 // because we were actually missing those intrinsics!
3253 AtomicRMWInst::BinOp BinOp =
3254 StringSwitch<AtomicRMWInst::BinOp>(Op)
3255 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3256 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3257 .StartsWith("exch.", AtomicRMWInst::Xchg)
3258 .StartsWith("max.", AtomicRMWInst::Max)
3259 .StartsWith("min.", AtomicRMWInst::Min)
3260 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3261 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3262 .StartsWith("and.", AtomicRMWInst::And)
3263 .StartsWith("or.", AtomicRMWInst::Or)
3264 .StartsWith("xor.", AtomicRMWInst::Xor)
3265 .Default(AtomicRMWInst::BAD_BINOP);
3266 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3267 "unexpected nvvm scoped atomic intrinsic");
3268 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3269 AtomicOrdering::Monotonic, SSID);
3270 }
3271 } else if (Name == "clz.ll") {
3272 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3273 Value *Arg = CI->getArgOperand(0);
3274 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3275 {Arg, Builder.getFalse()},
3276 /*FMFSource=*/nullptr, "ctlz");
3277 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3278 } else if (Name == "popc.ll") {
3279 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3280 // i64.
3281 Value *Arg = CI->getArgOperand(0);
3282 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3283 Arg, /*FMFSource=*/nullptr, "ctpop");
3284 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3285 } else if (Name == "h2f") {
3286 Value *Cast =
3287 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3288 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3289 } else if (Name.consume_front("bitcast.") &&
3290 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3291 Name == "d2ll")) {
3292 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3293 } else if (Name == "rotate.b32") {
3294 Value *Arg = CI->getOperand(0);
3295 Value *ShiftAmt = CI->getOperand(1);
3296 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3297 {Arg, Arg, ShiftAmt});
3298 } else if (Name == "rotate.b64") {
3299 Type *Int64Ty = Builder.getInt64Ty();
3300 Value *Arg = CI->getOperand(0);
3301 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3302 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3303 {Arg, Arg, ZExtShiftAmt});
3304 } else if (Name == "rotate.right.b64") {
3305 Type *Int64Ty = Builder.getInt64Ty();
3306 Value *Arg = CI->getOperand(0);
3307 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3308 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3309 {Arg, Arg, ZExtShiftAmt});
3310 } else if (Name == "swap.lo.hi.b64") {
3311 Type *Int64Ty = Builder.getInt64Ty();
3312 Value *Arg = CI->getOperand(0);
3313 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3314 {Arg, Arg, Builder.getInt64(32)});
3315 } else if ((Name.consume_front("ptr.gen.to.") &&
3316 consumeNVVMPtrAddrSpace(Name)) ||
3317 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3318 Name.starts_with(".to.gen"))) {
3319 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3320 } else if (Name.consume_front("ldg.global")) {
3321 Value *Ptr = CI->getArgOperand(0);
3322 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3323 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3324 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3325 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3326 MDNode *MD = MDNode::get(Builder.getContext(), {});
3327 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3328 return LD;
3329 } else if (Name == "tanh.approx.f32") {
3330 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3331 FastMathFlags FMF;
3332 FMF.setApproxFunc();
3333 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3334 FMF);
3335 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3336 Value *Arg =
3337 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3338 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3339 {}, {Arg});
3340 } else if (Name == "barrier") {
3341 Rep = Builder.CreateIntrinsic(
3342 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3343 {CI->getArgOperand(0), CI->getArgOperand(1)});
3344 } else if (Name == "barrier.sync") {
3345 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3346 {CI->getArgOperand(0)});
3347 } else if (Name == "barrier.sync.cnt") {
3348 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3349 {CI->getArgOperand(0), CI->getArgOperand(1)});
3350 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3351 Name == "barrier0.or") {
3352 Value *C = CI->getArgOperand(0);
3353 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3354
3355 Intrinsic::ID IID =
3356 StringSwitch<Intrinsic::ID>(Name)
3357 .Case("barrier0.popc",
3358 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3359 .Case("barrier0.and",
3360 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3361 .Case("barrier0.or",
3362 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3363 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3364 Rep = Builder.CreateZExt(Bar, CI->getType());
3365 } else {
3366 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3367 if (IID != Intrinsic::not_intrinsic &&
3368 isLegacyNVPTXBF16IntSignature(F, IID)) {
3369 rename(F);
3370 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3371 SmallVector<Value *, 2> Args;
3372 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3373 Value *Arg = CI->getArgOperand(I);
3374 Type *OldType = Arg->getType();
3375 Type *NewType = NewFn->getArg(I)->getType();
3376 Args.push_back(
3377 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3378 ? Builder.CreateBitCast(Arg, NewType)
3379 : Arg);
3380 }
3381 Rep = Builder.CreateCall(NewFn, Args);
3382 if (F->getReturnType()->isIntegerTy())
3383 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3384 }
3385 }
3386
3387 return Rep;
3388}
3389
3390static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3391 IRBuilder<> &Builder) {
3392 LLVMContext &C = F->getContext();
3393 Value *Rep = nullptr;
3394
3395 if (Name.starts_with("sse4a.movnt.")) {
3396 SmallVector<Metadata *, 1> Elts;
3397 Elts.push_back(
3398 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3399 MDNode *Node = MDNode::get(C, Elts);
3400
3401 Value *Arg0 = CI->getArgOperand(0);
3402 Value *Arg1 = CI->getArgOperand(1);
3403
3404 // Nontemporal (unaligned) store of the 0'th element of the float/double
3405 // vector.
3406 Value *Extract =
3407 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3408
3409 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3410 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3411 } else if (Name.starts_with("avx.movnt.") ||
3412 Name.starts_with("avx512.storent.")) {
3413 SmallVector<Metadata *, 1> Elts;
3414 Elts.push_back(
3415 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3416 MDNode *Node = MDNode::get(C, Elts);
3417
3418 Value *Arg0 = CI->getArgOperand(0);
3419 Value *Arg1 = CI->getArgOperand(1);
3420
3421 StoreInst *SI = Builder.CreateAlignedStore(
3422 Arg1, Arg0,
3423 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3424 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3425 } else if (Name == "sse2.storel.dq") {
3426 Value *Arg0 = CI->getArgOperand(0);
3427 Value *Arg1 = CI->getArgOperand(1);
3428
3429 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3430 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3431 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3432 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3433 } else if (Name.starts_with("sse.storeu.") ||
3434 Name.starts_with("sse2.storeu.") ||
3435 Name.starts_with("avx.storeu.")) {
3436 Value *Arg0 = CI->getArgOperand(0);
3437 Value *Arg1 = CI->getArgOperand(1);
3438 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3439 } else if (Name == "avx512.mask.store.ss") {
3440 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3441 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3442 Mask, false);
3443 } else if (Name.starts_with("avx512.mask.store")) {
3444 // "avx512.mask.storeu." or "avx512.mask.store."
3445 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3446 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3447 CI->getArgOperand(2), Aligned);
3448 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3449 // Upgrade packed integer vector compare intrinsics to compare instructions.
3450 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3451 bool CmpEq = Name[9] == 'e';
3452 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3453 CI->getArgOperand(0), CI->getArgOperand(1));
3454 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3455 } else if (Name.starts_with("avx512.broadcastm")) {
3456 Type *ExtTy = Type::getInt32Ty(C);
3457 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3458 ExtTy = Type::getInt64Ty(C);
3459 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3460 ExtTy->getPrimitiveSizeInBits();
3461 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3462 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3463 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3464 Value *Vec = CI->getArgOperand(0);
3465 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3466 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3467 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3468 } else if (Name.starts_with("avx.sqrt.p") ||
3469 Name.starts_with("sse2.sqrt.p") ||
3470 Name.starts_with("sse.sqrt.p")) {
3471 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3472 {CI->getArgOperand(0)});
3473 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3474 if (CI->arg_size() == 4 &&
3475 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3476 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3477 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3478 : Intrinsic::x86_avx512_sqrt_pd_512;
3479
3480 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3481 Rep = Builder.CreateIntrinsic(IID, Args);
3482 } else {
3483 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3484 {CI->getArgOperand(0)});
3485 }
3486 Rep =
3487 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3488 } else if (Name.starts_with("avx512.ptestm") ||
3489 Name.starts_with("avx512.ptestnm")) {
3490 Value *Op0 = CI->getArgOperand(0);
3491 Value *Op1 = CI->getArgOperand(1);
3492 Value *Mask = CI->getArgOperand(2);
3493 Rep = Builder.CreateAnd(Op0, Op1);
3494 llvm::Type *Ty = Op0->getType();
3495 Value *Zero = llvm::Constant::getNullValue(Ty);
3496 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3497 ? ICmpInst::ICMP_NE
3498 : ICmpInst::ICMP_EQ;
3499 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3500 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3501 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3502 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3503 ->getNumElements();
3504 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3505 Rep =
3506 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3507 } else if (Name.starts_with("avx512.kunpck")) {
3508 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3509 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3510 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3511 int Indices[64];
3512 for (unsigned i = 0; i != NumElts; ++i)
3513 Indices[i] = i;
3514
3515 // First extract half of each vector. This gives better codegen than
3516 // doing it in a single shuffle.
3517 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3518 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3519 // Concat the vectors.
3520 // NOTE: Operands have to be swapped to match intrinsic definition.
3521 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3522 Rep = Builder.CreateBitCast(Rep, CI->getType());
3523 } else if (Name == "avx512.kand.w") {
3524 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3525 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3526 Rep = Builder.CreateAnd(LHS, RHS);
3527 Rep = Builder.CreateBitCast(Rep, CI->getType());
3528 } else if (Name == "avx512.kandn.w") {
3529 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3530 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3531 LHS = Builder.CreateNot(LHS);
3532 Rep = Builder.CreateAnd(LHS, RHS);
3533 Rep = Builder.CreateBitCast(Rep, CI->getType());
3534 } else if (Name == "avx512.kor.w") {
3535 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3536 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3537 Rep = Builder.CreateOr(LHS, RHS);
3538 Rep = Builder.CreateBitCast(Rep, CI->getType());
3539 } else if (Name == "avx512.kxor.w") {
3540 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3541 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3542 Rep = Builder.CreateXor(LHS, RHS);
3543 Rep = Builder.CreateBitCast(Rep, CI->getType());
3544 } else if (Name == "avx512.kxnor.w") {
3545 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3546 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3547 LHS = Builder.CreateNot(LHS);
3548 Rep = Builder.CreateXor(LHS, RHS);
3549 Rep = Builder.CreateBitCast(Rep, CI->getType());
3550 } else if (Name == "avx512.knot.w") {
3551 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3552 Rep = Builder.CreateNot(Rep);
3553 Rep = Builder.CreateBitCast(Rep, CI->getType());
3554 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3555 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3556 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3557 Rep = Builder.CreateOr(LHS, RHS);
3558 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3559 Value *C;
3560 if (Name[14] == 'c')
3561 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3562 else
3563 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3564 Rep = Builder.CreateICmpEQ(Rep, C);
3565 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3566 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3567 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3568 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3569 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3570 Type *I32Ty = Type::getInt32Ty(C);
3571 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3572 ConstantInt::get(I32Ty, 0));
3573 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3574 ConstantInt::get(I32Ty, 0));
3575 Value *EltOp;
3576 if (Name.contains(".add."))
3577 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3578 else if (Name.contains(".sub."))
3579 EltOp = Builder.CreateFSub(Elt0, Elt1);
3580 else if (Name.contains(".mul."))
3581 EltOp = Builder.CreateFMul(Elt0, Elt1);
3582 else
3583 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3584 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3585 ConstantInt::get(I32Ty, 0));
3586 } else if (Name.starts_with("avx512.mask.pcmp")) {
3587 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3588 bool CmpEq = Name[16] == 'e';
3589 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3590 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3591 Type *OpTy = CI->getArgOperand(0)->getType();
3592 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3593 Intrinsic::ID IID;
3594 switch (VecWidth) {
3595 default:
3596 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3597 break;
3598 case 128:
3599 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3600 break;
3601 case 256:
3602 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3603 break;
3604 case 512:
3605 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3606 break;
3607 }
3608
3609 Rep =
3610 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3611 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3612 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3613 Type *OpTy = CI->getArgOperand(0)->getType();
3614 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3615 unsigned EltWidth = OpTy->getScalarSizeInBits();
3616 Intrinsic::ID IID;
3617 if (VecWidth == 128 && EltWidth == 32)
3618 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3619 else if (VecWidth == 256 && EltWidth == 32)
3620 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3621 else if (VecWidth == 512 && EltWidth == 32)
3622 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3623 else if (VecWidth == 128 && EltWidth == 64)
3624 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3625 else if (VecWidth == 256 && EltWidth == 64)
3626 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3627 else if (VecWidth == 512 && EltWidth == 64)
3628 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3629 else
3630 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3631
3632 Rep =
3633 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3634 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3635 } else if (Name.starts_with("avx512.cmp.p")) {
3636 SmallVector<Value *, 4> Args(CI->args());
3637 Type *OpTy = Args[0]->getType();
3638 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3639 unsigned EltWidth = OpTy->getScalarSizeInBits();
3640 Intrinsic::ID IID;
3641 if (VecWidth == 128 && EltWidth == 32)
3642 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3643 else if (VecWidth == 256 && EltWidth == 32)
3644 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3645 else if (VecWidth == 512 && EltWidth == 32)
3646 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3647 else if (VecWidth == 128 && EltWidth == 64)
3648 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3649 else if (VecWidth == 256 && EltWidth == 64)
3650 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3651 else if (VecWidth == 512 && EltWidth == 64)
3652 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3653 else
3654 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3655
3656 Value *Mask = Constant::getAllOnesValue(CI->getType());
3657 if (VecWidth == 512)
3658 std::swap(Mask, Args.back());
3659 Args.push_back(Mask);
3660
3661 Rep = Builder.CreateIntrinsic(IID, Args);
3662 } else if (Name.starts_with("avx512.mask.cmp.")) {
3663 // Integer compare intrinsics.
3664 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3665 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3666 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3667 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3668 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3669 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3670 Name.starts_with("avx512.cvtw2mask.") ||
3671 Name.starts_with("avx512.cvtd2mask.") ||
3672 Name.starts_with("avx512.cvtq2mask.")) {
3673 Value *Op = CI->getArgOperand(0);
3674 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3675 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3676 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3677 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3678 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3679 Name.starts_with("avx512.mask.pabs")) {
3680 Rep = upgradeAbs(Builder, *CI);
3681 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3682 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3683 Name.starts_with("avx512.mask.pmaxs")) {
3684 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3685 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3686 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3687 Name.starts_with("avx512.mask.pmaxu")) {
3688 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3689 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3690 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3691 Name.starts_with("avx512.mask.pmins")) {
3692 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3693 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3694 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3695 Name.starts_with("avx512.mask.pminu")) {
3696 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3697 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3698 Name == "avx512.pmulu.dq.512" ||
3699 Name.starts_with("avx512.mask.pmulu.dq.")) {
3700 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3701 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3702 Name == "avx512.pmul.dq.512" ||
3703 Name.starts_with("avx512.mask.pmul.dq.")) {
3704 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3705 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3706 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3707 Rep =
3708 Builder.CreateSIToFP(CI->getArgOperand(1),
3709 cast<VectorType>(CI->getType())->getElementType());
3710 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3711 } else if (Name == "avx512.cvtusi2sd") {
3712 Rep =
3713 Builder.CreateUIToFP(CI->getArgOperand(1),
3714 cast<VectorType>(CI->getType())->getElementType());
3715 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3716 } else if (Name == "sse2.cvtss2sd") {
3717 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3718 Rep = Builder.CreateFPExt(
3719 Rep, cast<VectorType>(CI->getType())->getElementType());
3720 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3721 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3722 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3723 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3724 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3725 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3726 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3727 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3728 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3729 Name == "avx512.mask.cvtqq2ps.256" ||
3730 Name == "avx512.mask.cvtqq2ps.512" ||
3731 Name == "avx512.mask.cvtuqq2ps.256" ||
3732 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3733 Name == "avx.cvt.ps2.pd.256" ||
3734 Name == "avx512.mask.cvtps2pd.128" ||
3735 Name == "avx512.mask.cvtps2pd.256") {
3736 auto *DstTy = cast<FixedVectorType>(CI->getType());
3737 Rep = CI->getArgOperand(0);
3738 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3739
3740 unsigned NumDstElts = DstTy->getNumElements();
3741 if (NumDstElts < SrcTy->getNumElements()) {
3742 assert(NumDstElts == 2 && "Unexpected vector size");
3743 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3744 }
3745
3746 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3747 bool IsUnsigned = Name.contains("cvtu");
3748 if (IsPS2PD)
3749 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3750 else if (CI->arg_size() == 4 &&
3751 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3752 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3753 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3754 : Intrinsic::x86_avx512_sitofp_round;
3755 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3756 {Rep, CI->getArgOperand(3)});
3757 } else {
3758 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3759 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3760 }
3761
3762 if (CI->arg_size() >= 3)
3763 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3764 CI->getArgOperand(1));
3765 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3766 Name.starts_with("vcvtph2ps.")) {
3767 auto *DstTy = cast<FixedVectorType>(CI->getType());
3768 Rep = CI->getArgOperand(0);
3769 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3770 unsigned NumDstElts = DstTy->getNumElements();
3771 if (NumDstElts != SrcTy->getNumElements()) {
3772 assert(NumDstElts == 4 && "Unexpected vector size");
3773 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3774 }
3775 Rep = Builder.CreateBitCast(
3776 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3777 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3778 if (CI->arg_size() >= 3)
3779 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3780 CI->getArgOperand(1));
3781 } else if (Name.starts_with("avx512.mask.load")) {
3782 // "avx512.mask.loadu." or "avx512.mask.load."
3783 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3784 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3785 CI->getArgOperand(2), Aligned);
3786 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3787 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3788 auto *PtrTy = CI->getOperand(0)->getType();
3789 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3790 ResultTy->getNumElements());
3791 Rep = Builder.CreateIntrinsic(
3792 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3793 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3794 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3795 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3796 auto *PtrTy = CI->getArgOperand(0)->getType();
3797 Value *MaskVec =
3798 getX86MaskVec(Builder, CI->getArgOperand(2),
3799 cast<FixedVectorType>(ResultTy)->getNumElements());
3800 Rep = Builder.CreateIntrinsic(
3801 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3802 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3803 } else if (Name.starts_with("avx512.mask.compress.") ||
3804 Name.starts_with("avx512.mask.expand.")) {
3805 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3806
3807 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3808 ResultTy->getNumElements());
3809
3810 bool IsCompress = Name[12] == 'c';
3811 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3812 : Intrinsic::x86_avx512_mask_expand;
3813 Rep = Builder.CreateIntrinsic(
3814 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3815 } else if (Name.starts_with("xop.vpcom")) {
3816 bool IsSigned;
3817 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3818 Name.ends_with("uq"))
3819 IsSigned = false;
3820 else if (Name.ends_with("b") || Name.ends_with("w") ||
3821 Name.ends_with("d") || Name.ends_with("q"))
3822 IsSigned = true;
3823 else
3824 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3825
3826 unsigned Imm;
3827 if (CI->arg_size() == 3) {
3828 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3829 } else {
3830 Name = Name.substr(9); // strip off "xop.vpcom"
3831 if (Name.starts_with("lt"))
3832 Imm = 0;
3833 else if (Name.starts_with("le"))
3834 Imm = 1;
3835 else if (Name.starts_with("gt"))
3836 Imm = 2;
3837 else if (Name.starts_with("ge"))
3838 Imm = 3;
3839 else if (Name.starts_with("eq"))
3840 Imm = 4;
3841 else if (Name.starts_with("ne"))
3842 Imm = 5;
3843 else if (Name.starts_with("false"))
3844 Imm = 6;
3845 else if (Name.starts_with("true"))
3846 Imm = 7;
3847 else
3848 llvm_unreachable("Unknown condition");
3849 }
3850
3851 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3852 } else if (Name.starts_with("xop.vpcmov")) {
3853 Value *Sel = CI->getArgOperand(2);
3854 Value *NotSel = Builder.CreateNot(Sel);
3855 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3856 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3857 Rep = Builder.CreateOr(Sel0, Sel1);
3858 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3859 Name.starts_with("avx512.mask.prol")) {
3860 Rep = upgradeX86Rotate(Builder, *CI, false);
3861 } else if (Name.starts_with("avx512.pror") ||
3862 Name.starts_with("avx512.mask.pror")) {
3863 Rep = upgradeX86Rotate(Builder, *CI, true);
3864 } else if (Name.starts_with("avx512.vpshld.") ||
3865 Name.starts_with("avx512.mask.vpshld") ||
3866 Name.starts_with("avx512.maskz.vpshld")) {
3867 bool ZeroMask = Name[11] == 'z';
3868 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3869 } else if (Name.starts_with("avx512.vpshrd.") ||
3870 Name.starts_with("avx512.mask.vpshrd") ||
3871 Name.starts_with("avx512.maskz.vpshrd")) {
3872 bool ZeroMask = Name[11] == 'z';
3873 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3874 } else if (Name == "sse42.crc32.64.8") {
3875 Value *Trunc0 =
3876 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3877 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3878 {Trunc0, CI->getArgOperand(1)});
3879 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3880 } else if (Name.starts_with("avx.vbroadcast.s") ||
3881 Name.starts_with("avx512.vbroadcast.s")) {
3882 // Replace broadcasts with a series of insertelements.
3883 auto *VecTy = cast<FixedVectorType>(CI->getType());
3884 Type *EltTy = VecTy->getElementType();
3885 unsigned EltNum = VecTy->getNumElements();
3886 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3887 Type *I32Ty = Type::getInt32Ty(C);
3888 Rep = PoisonValue::get(VecTy);
3889 for (unsigned I = 0; I < EltNum; ++I)
3890 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3891 } else if (Name.starts_with("sse41.pmovsx") ||
3892 Name.starts_with("sse41.pmovzx") ||
3893 Name.starts_with("avx2.pmovsx") ||
3894 Name.starts_with("avx2.pmovzx") ||
3895 Name.starts_with("avx512.mask.pmovsx") ||
3896 Name.starts_with("avx512.mask.pmovzx")) {
3897 auto *DstTy = cast<FixedVectorType>(CI->getType());
3898 unsigned NumDstElts = DstTy->getNumElements();
3899
3900 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3901 SmallVector<int, 8> ShuffleMask(NumDstElts);
3902 for (unsigned i = 0; i != NumDstElts; ++i)
3903 ShuffleMask[i] = i;
3904
3905 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3906
3907 bool DoSext = Name.contains("pmovsx");
3908 Rep =
3909 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3910 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3911 if (CI->arg_size() == 3)
3912 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3913 CI->getArgOperand(1));
3914 } else if (Name == "avx512.mask.pmov.qd.256" ||
3915 Name == "avx512.mask.pmov.qd.512" ||
3916 Name == "avx512.mask.pmov.wb.256" ||
3917 Name == "avx512.mask.pmov.wb.512") {
3918 Type *Ty = CI->getArgOperand(1)->getType();
3919 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3920 Rep =
3921 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3922 } else if (Name.starts_with("avx.vbroadcastf128") ||
3923 Name == "avx2.vbroadcasti128") {
3924 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3925 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3926 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3927 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3928 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3929 if (NumSrcElts == 2)
3930 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3931 else
3932 Rep = Builder.CreateShuffleVector(Load,
3933 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3934 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3935 Name.starts_with("avx512.mask.shuf.f")) {
3936 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3937 Type *VT = CI->getType();
3938 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3939 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3940 unsigned ControlBitsMask = NumLanes - 1;
3941 unsigned NumControlBits = NumLanes / 2;
3942 SmallVector<int, 8> ShuffleMask(0);
3943
3944 for (unsigned l = 0; l != NumLanes; ++l) {
3945 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3946 // We actually need the other source.
3947 if (l >= NumLanes / 2)
3948 LaneMask += NumLanes;
3949 for (unsigned i = 0; i != NumElementsInLane; ++i)
3950 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3951 }
3952 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3953 CI->getArgOperand(1), ShuffleMask);
3954 Rep =
3955 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3956 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3957 Name.starts_with("avx512.mask.broadcasti")) {
3958 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3959 ->getNumElements();
3960 unsigned NumDstElts =
3961 cast<FixedVectorType>(CI->getType())->getNumElements();
3962
3963 SmallVector<int, 8> ShuffleMask(NumDstElts);
3964 for (unsigned i = 0; i != NumDstElts; ++i)
3965 ShuffleMask[i] = i % NumSrcElts;
3966
3967 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3968 CI->getArgOperand(0), ShuffleMask);
3969 Rep =
3970 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3971 } else if (Name.starts_with("avx2.pbroadcast") ||
3972 Name.starts_with("avx2.vbroadcast") ||
3973 Name.starts_with("avx512.pbroadcast") ||
3974 Name.starts_with("avx512.mask.broadcast.s")) {
3975 // Replace vp?broadcasts with a vector shuffle.
3976 Value *Op = CI->getArgOperand(0);
3977 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3978 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3979 SmallVector<int, 8> M;
3980 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3981 Rep = Builder.CreateShuffleVector(Op, M);
3982
3983 if (CI->arg_size() == 3)
3984 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3985 CI->getArgOperand(1));
3986 } else if (Name.starts_with("sse2.padds.") ||
3987 Name.starts_with("avx2.padds.") ||
3988 Name.starts_with("avx512.padds.") ||
3989 Name.starts_with("avx512.mask.padds.")) {
3990 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3991 } else if (Name.starts_with("sse2.psubs.") ||
3992 Name.starts_with("avx2.psubs.") ||
3993 Name.starts_with("avx512.psubs.") ||
3994 Name.starts_with("avx512.mask.psubs.")) {
3995 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3996 } else if (Name.starts_with("sse2.paddus.") ||
3997 Name.starts_with("avx2.paddus.") ||
3998 Name.starts_with("avx512.mask.paddus.")) {
3999 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4000 } else if (Name.starts_with("sse2.psubus.") ||
4001 Name.starts_with("avx2.psubus.") ||
4002 Name.starts_with("avx512.mask.psubus.")) {
4003 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4004 } else if (Name.starts_with("avx512.mask.palignr.")) {
4005 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4006 CI->getArgOperand(1), CI->getArgOperand(2),
4007 CI->getArgOperand(3), CI->getArgOperand(4),
4008 false);
4009 } else if (Name.starts_with("avx512.mask.valign.")) {
4010 Rep = upgradeX86ALIGNIntrinsics(
4011 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4012 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4013 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4014 // 128/256-bit shift left specified in bits.
4015 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4016 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4017 Shift / 8); // Shift is in bits.
4018 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4019 // 128/256-bit shift right specified in bits.
4020 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4021 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4022 Shift / 8); // Shift is in bits.
4023 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4024 Name == "avx512.psll.dq.512") {
4025 // 128/256/512-bit shift left specified in bytes.
4026 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4027 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4028 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4029 Name == "avx512.psrl.dq.512") {
4030 // 128/256/512-bit shift right specified in bytes.
4031 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4032 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4033 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4034 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4035 Name.starts_with("avx2.pblendd.")) {
4036 Value *Op0 = CI->getArgOperand(0);
4037 Value *Op1 = CI->getArgOperand(1);
4038 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4039 auto *VecTy = cast<FixedVectorType>(CI->getType());
4040 unsigned NumElts = VecTy->getNumElements();
4041
4042 SmallVector<int, 16> Idxs(NumElts);
4043 for (unsigned i = 0; i != NumElts; ++i)
4044 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4045
4046 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4047 } else if (Name.starts_with("avx.vinsertf128.") ||
4048 Name == "avx2.vinserti128" ||
4049 Name.starts_with("avx512.mask.insert")) {
4050 Value *Op0 = CI->getArgOperand(0);
4051 Value *Op1 = CI->getArgOperand(1);
4052 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4053 unsigned DstNumElts =
4054 cast<FixedVectorType>(CI->getType())->getNumElements();
4055 unsigned SrcNumElts =
4056 cast<FixedVectorType>(Op1->getType())->getNumElements();
4057 unsigned Scale = DstNumElts / SrcNumElts;
4058
4059 // Mask off the high bits of the immediate value; hardware ignores those.
4060 Imm = Imm % Scale;
4061
4062 // Extend the second operand into a vector the size of the destination.
4063 SmallVector<int, 8> Idxs(DstNumElts);
4064 for (unsigned i = 0; i != SrcNumElts; ++i)
4065 Idxs[i] = i;
4066 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4067 Idxs[i] = SrcNumElts;
4068 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4069
4070 // Insert the second operand into the first operand.
4071
4072 // Note that there is no guarantee that instruction lowering will actually
4073 // produce a vinsertf128 instruction for the created shuffles. In
4074 // particular, the 0 immediate case involves no lane changes, so it can
4075 // be handled as a blend.
4076
4077 // Example of shuffle mask for 32-bit elements:
4078 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4079 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4080
4081 // First fill with identify mask.
4082 for (unsigned i = 0; i != DstNumElts; ++i)
4083 Idxs[i] = i;
4084 // Then replace the elements where we need to insert.
4085 for (unsigned i = 0; i != SrcNumElts; ++i)
4086 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4087 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4088
4089 // If the intrinsic has a mask operand, handle that.
4090 if (CI->arg_size() == 5)
4091 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4092 CI->getArgOperand(3));
4093 } else if (Name.starts_with("avx.vextractf128.") ||
4094 Name == "avx2.vextracti128" ||
4095 Name.starts_with("avx512.mask.vextract")) {
4096 Value *Op0 = CI->getArgOperand(0);
4097 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4098 unsigned DstNumElts =
4099 cast<FixedVectorType>(CI->getType())->getNumElements();
4100 unsigned SrcNumElts =
4101 cast<FixedVectorType>(Op0->getType())->getNumElements();
4102 unsigned Scale = SrcNumElts / DstNumElts;
4103
4104 // Mask off the high bits of the immediate value; hardware ignores those.
4105 Imm = Imm % Scale;
4106
4107 // Get indexes for the subvector of the input vector.
4108 SmallVector<int, 8> Idxs(DstNumElts);
4109 for (unsigned i = 0; i != DstNumElts; ++i) {
4110 Idxs[i] = i + (Imm * DstNumElts);
4111 }
4112 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4113
4114 // If the intrinsic has a mask operand, handle that.
4115 if (CI->arg_size() == 4)
4116 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4117 CI->getArgOperand(2));
4118 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4119 Name.starts_with("avx512.mask.perm.di.")) {
4120 Value *Op0 = CI->getArgOperand(0);
4121 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4122 auto *VecTy = cast<FixedVectorType>(CI->getType());
4123 unsigned NumElts = VecTy->getNumElements();
4124
4125 SmallVector<int, 8> Idxs(NumElts);
4126 for (unsigned i = 0; i != NumElts; ++i)
4127 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4128
4129 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4130
4131 if (CI->arg_size() == 4)
4132 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4133 CI->getArgOperand(2));
4134 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4135 // The immediate permute control byte looks like this:
4136 // [1:0] - select 128 bits from sources for low half of destination
4137 // [2] - ignore
4138 // [3] - zero low half of destination
4139 // [5:4] - select 128 bits from sources for high half of destination
4140 // [6] - ignore
4141 // [7] - zero high half of destination
4142
4143 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4144
4145 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4146 unsigned HalfSize = NumElts / 2;
4147 SmallVector<int, 8> ShuffleMask(NumElts);
4148
4149 // Determine which operand(s) are actually in use for this instruction.
4150 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4151 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4152
4153 // If needed, replace operands based on zero mask.
4154 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4155 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4156
4157 // Permute low half of result.
4158 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4159 for (unsigned i = 0; i < HalfSize; ++i)
4160 ShuffleMask[i] = StartIndex + i;
4161
4162 // Permute high half of result.
4163 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4164 for (unsigned i = 0; i < HalfSize; ++i)
4165 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4166
4167 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4168
4169 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4170 Name.starts_with("avx512.mask.vpermil.p") ||
4171 Name.starts_with("avx512.mask.pshuf.d.")) {
4172 Value *Op0 = CI->getArgOperand(0);
4173 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4174 auto *VecTy = cast<FixedVectorType>(CI->getType());
4175 unsigned NumElts = VecTy->getNumElements();
4176 // Calculate the size of each index in the immediate.
4177 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4178 unsigned IdxMask = ((1 << IdxSize) - 1);
4179
4180 SmallVector<int, 8> Idxs(NumElts);
4181 // Lookup the bits for this element, wrapping around the immediate every
4182 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4183 // to offset by the first index of each group.
4184 for (unsigned i = 0; i != NumElts; ++i)
4185 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4186
4187 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4188
4189 if (CI->arg_size() == 4)
4190 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4191 CI->getArgOperand(2));
4192 } else if (Name == "sse2.pshufl.w" ||
4193 Name.starts_with("avx512.mask.pshufl.w.")) {
4194 Value *Op0 = CI->getArgOperand(0);
4195 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4196 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4197
4198 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4199 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4200
4201 SmallVector<int, 16> Idxs(NumElts);
4202 for (unsigned l = 0; l != NumElts; l += 8) {
4203 for (unsigned i = 0; i != 4; ++i)
4204 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4205 for (unsigned i = 4; i != 8; ++i)
4206 Idxs[i + l] = i + l;
4207 }
4208
4209 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4210
4211 if (CI->arg_size() == 4)
4212 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4213 CI->getArgOperand(2));
4214 } else if (Name == "sse2.pshufh.w" ||
4215 Name.starts_with("avx512.mask.pshufh.w.")) {
4216 Value *Op0 = CI->getArgOperand(0);
4217 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4218 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4219
4220 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4221 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4222
4223 SmallVector<int, 16> Idxs(NumElts);
4224 for (unsigned l = 0; l != NumElts; l += 8) {
4225 for (unsigned i = 0; i != 4; ++i)
4226 Idxs[i + l] = i + l;
4227 for (unsigned i = 0; i != 4; ++i)
4228 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4229 }
4230
4231 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4232
4233 if (CI->arg_size() == 4)
4234 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4235 CI->getArgOperand(2));
4236 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4237 Value *Op0 = CI->getArgOperand(0);
4238 Value *Op1 = CI->getArgOperand(1);
4239 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4240 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4241
4242 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4243 unsigned HalfLaneElts = NumLaneElts / 2;
4244
4245 SmallVector<int, 16> Idxs(NumElts);
4246 for (unsigned i = 0; i != NumElts; ++i) {
4247 // Base index is the starting element of the lane.
4248 Idxs[i] = i - (i % NumLaneElts);
4249 // If we are half way through the lane switch to the other source.
4250 if ((i % NumLaneElts) >= HalfLaneElts)
4251 Idxs[i] += NumElts;
4252 // Now select the specific element. By adding HalfLaneElts bits from
4253 // the immediate. Wrapping around the immediate every 8-bits.
4254 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4255 }
4256
4257 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4258
4259 Rep =
4260 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4261 } else if (Name.starts_with("avx512.mask.movddup") ||
4262 Name.starts_with("avx512.mask.movshdup") ||
4263 Name.starts_with("avx512.mask.movsldup")) {
4264 Value *Op0 = CI->getArgOperand(0);
4265 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4266 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4267
4268 unsigned Offset = 0;
4269 if (Name.starts_with("avx512.mask.movshdup."))
4270 Offset = 1;
4271
4272 SmallVector<int, 16> Idxs(NumElts);
4273 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4274 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4275 Idxs[i + l + 0] = i + l + Offset;
4276 Idxs[i + l + 1] = i + l + Offset;
4277 }
4278
4279 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4280
4281 Rep =
4282 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4283 } else if (Name.starts_with("avx512.mask.punpckl") ||
4284 Name.starts_with("avx512.mask.unpckl.")) {
4285 Value *Op0 = CI->getArgOperand(0);
4286 Value *Op1 = CI->getArgOperand(1);
4287 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4288 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4289
4290 SmallVector<int, 64> Idxs(NumElts);
4291 for (int l = 0; l != NumElts; l += NumLaneElts)
4292 for (int i = 0; i != NumLaneElts; ++i)
4293 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4294
4295 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4296
4297 Rep =
4298 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4299 } else if (Name.starts_with("avx512.mask.punpckh") ||
4300 Name.starts_with("avx512.mask.unpckh.")) {
4301 Value *Op0 = CI->getArgOperand(0);
4302 Value *Op1 = CI->getArgOperand(1);
4303 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4304 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4305
4306 SmallVector<int, 64> Idxs(NumElts);
4307 for (int l = 0; l != NumElts; l += NumLaneElts)
4308 for (int i = 0; i != NumLaneElts; ++i)
4309 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4310
4311 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4312
4313 Rep =
4314 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4315 } else if (Name.starts_with("avx512.mask.and.") ||
4316 Name.starts_with("avx512.mask.pand.")) {
4317 VectorType *FTy = cast<VectorType>(CI->getType());
4318 VectorType *ITy = VectorType::getInteger(FTy);
4319 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4320 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4321 Rep = Builder.CreateBitCast(Rep, FTy);
4322 Rep =
4323 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4324 } else if (Name.starts_with("avx512.mask.andn.") ||
4325 Name.starts_with("avx512.mask.pandn.")) {
4326 VectorType *FTy = cast<VectorType>(CI->getType());
4327 VectorType *ITy = VectorType::getInteger(FTy);
4328 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4329 Rep = Builder.CreateAnd(Rep,
4330 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4331 Rep = Builder.CreateBitCast(Rep, FTy);
4332 Rep =
4333 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4334 } else if (Name.starts_with("avx512.mask.or.") ||
4335 Name.starts_with("avx512.mask.por.")) {
4336 VectorType *FTy = cast<VectorType>(CI->getType());
4337 VectorType *ITy = VectorType::getInteger(FTy);
4338 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4339 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4340 Rep = Builder.CreateBitCast(Rep, FTy);
4341 Rep =
4342 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4343 } else if (Name.starts_with("avx512.mask.xor.") ||
4344 Name.starts_with("avx512.mask.pxor.")) {
4345 VectorType *FTy = cast<VectorType>(CI->getType());
4346 VectorType *ITy = VectorType::getInteger(FTy);
4347 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4348 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4349 Rep = Builder.CreateBitCast(Rep, FTy);
4350 Rep =
4351 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4352 } else if (Name.starts_with("avx512.mask.padd.")) {
4353 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4354 Rep =
4355 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4356 } else if (Name.starts_with("avx512.mask.psub.")) {
4357 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4358 Rep =
4359 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4360 } else if (Name.starts_with("avx512.mask.pmull.")) {
4361 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4362 Rep =
4363 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4364 } else if (Name.starts_with("avx512.mask.add.p")) {
4365 if (Name.ends_with(".512")) {
4366 Intrinsic::ID IID;
4367 if (Name[17] == 's')
4368 IID = Intrinsic::x86_avx512_add_ps_512;
4369 else
4370 IID = Intrinsic::x86_avx512_add_pd_512;
4371
4372 Rep = Builder.CreateIntrinsic(
4373 IID,
4374 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4375 } else {
4376 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4377 }
4378 Rep =
4379 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4380 } else if (Name.starts_with("avx512.mask.div.p")) {
4381 if (Name.ends_with(".512")) {
4382 Intrinsic::ID IID;
4383 if (Name[17] == 's')
4384 IID = Intrinsic::x86_avx512_div_ps_512;
4385 else
4386 IID = Intrinsic::x86_avx512_div_pd_512;
4387
4388 Rep = Builder.CreateIntrinsic(
4389 IID,
4390 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4391 } else {
4392 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4393 }
4394 Rep =
4395 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4396 } else if (Name.starts_with("avx512.mask.mul.p")) {
4397 if (Name.ends_with(".512")) {
4398 Intrinsic::ID IID;
4399 if (Name[17] == 's')
4400 IID = Intrinsic::x86_avx512_mul_ps_512;
4401 else
4402 IID = Intrinsic::x86_avx512_mul_pd_512;
4403
4404 Rep = Builder.CreateIntrinsic(
4405 IID,
4406 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4407 } else {
4408 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4409 }
4410 Rep =
4411 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4412 } else if (Name.starts_with("avx512.mask.sub.p")) {
4413 if (Name.ends_with(".512")) {
4414 Intrinsic::ID IID;
4415 if (Name[17] == 's')
4416 IID = Intrinsic::x86_avx512_sub_ps_512;
4417 else
4418 IID = Intrinsic::x86_avx512_sub_pd_512;
4419
4420 Rep = Builder.CreateIntrinsic(
4421 IID,
4422 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4423 } else {
4424 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4425 }
4426 Rep =
4427 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4428 } else if ((Name.starts_with("avx512.mask.max.p") ||
4429 Name.starts_with("avx512.mask.min.p")) &&
4430 Name.drop_front(18) == ".512") {
4431 bool IsDouble = Name[17] == 'd';
4432 bool IsMin = Name[13] == 'i';
4433 static const Intrinsic::ID MinMaxTbl[2][2] = {
4434 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4435 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4436 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4437
4438 Rep = Builder.CreateIntrinsic(
4439 IID,
4440 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4441 Rep =
4442 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4443 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4444 Rep =
4445 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4446 {CI->getArgOperand(0), Builder.getInt1(false)});
4447 Rep =
4448 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4449 } else if (Name.starts_with("avx512.mask.psll")) {
4450 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4451 bool IsVariable = Name[16] == 'v';
4452 char Size = Name[16] == '.' ? Name[17]
4453 : Name[17] == '.' ? Name[18]
4454 : Name[18] == '.' ? Name[19]
4455 : Name[20];
4456
4457 Intrinsic::ID IID;
4458 if (IsVariable && Name[17] != '.') {
4459 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4460 IID = Intrinsic::x86_avx2_psllv_q;
4461 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4462 IID = Intrinsic::x86_avx2_psllv_q_256;
4463 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4464 IID = Intrinsic::x86_avx2_psllv_d;
4465 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4466 IID = Intrinsic::x86_avx2_psllv_d_256;
4467 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4468 IID = Intrinsic::x86_avx512_psllv_w_128;
4469 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4470 IID = Intrinsic::x86_avx512_psllv_w_256;
4471 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4472 IID = Intrinsic::x86_avx512_psllv_w_512;
4473 else
4474 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4475 } else if (Name.ends_with(".128")) {
4476 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4477 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4478 : Intrinsic::x86_sse2_psll_d;
4479 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4480 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4481 : Intrinsic::x86_sse2_psll_q;
4482 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4483 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4484 : Intrinsic::x86_sse2_psll_w;
4485 else
4486 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4487 } else if (Name.ends_with(".256")) {
4488 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4489 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4490 : Intrinsic::x86_avx2_psll_d;
4491 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4492 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4493 : Intrinsic::x86_avx2_psll_q;
4494 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4495 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4496 : Intrinsic::x86_avx2_psll_w;
4497 else
4498 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4499 } else {
4500 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4501 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4502 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4503 : Intrinsic::x86_avx512_psll_d_512;
4504 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4505 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4506 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4507 : Intrinsic::x86_avx512_psll_q_512;
4508 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4509 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4510 : Intrinsic::x86_avx512_psll_w_512;
4511 else
4512 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4513 }
4514
4515 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4516 } else if (Name.starts_with("avx512.mask.psrl")) {
4517 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4518 bool IsVariable = Name[16] == 'v';
4519 char Size = Name[16] == '.' ? Name[17]
4520 : Name[17] == '.' ? Name[18]
4521 : Name[18] == '.' ? Name[19]
4522 : Name[20];
4523
4524 Intrinsic::ID IID;
4525 if (IsVariable && Name[17] != '.') {
4526 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4527 IID = Intrinsic::x86_avx2_psrlv_q;
4528 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4529 IID = Intrinsic::x86_avx2_psrlv_q_256;
4530 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4531 IID = Intrinsic::x86_avx2_psrlv_d;
4532 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4533 IID = Intrinsic::x86_avx2_psrlv_d_256;
4534 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4535 IID = Intrinsic::x86_avx512_psrlv_w_128;
4536 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4537 IID = Intrinsic::x86_avx512_psrlv_w_256;
4538 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4539 IID = Intrinsic::x86_avx512_psrlv_w_512;
4540 else
4541 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4542 } else if (Name.ends_with(".128")) {
4543 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4544 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4545 : Intrinsic::x86_sse2_psrl_d;
4546 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4547 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4548 : Intrinsic::x86_sse2_psrl_q;
4549 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4550 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4551 : Intrinsic::x86_sse2_psrl_w;
4552 else
4553 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4554 } else if (Name.ends_with(".256")) {
4555 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4556 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4557 : Intrinsic::x86_avx2_psrl_d;
4558 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4559 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4560 : Intrinsic::x86_avx2_psrl_q;
4561 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4562 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4563 : Intrinsic::x86_avx2_psrl_w;
4564 else
4565 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4566 } else {
4567 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4568 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4569 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4570 : Intrinsic::x86_avx512_psrl_d_512;
4571 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4572 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4573 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4574 : Intrinsic::x86_avx512_psrl_q_512;
4575 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4576 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4577 : Intrinsic::x86_avx512_psrl_w_512;
4578 else
4579 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4580 }
4581
4582 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4583 } else if (Name.starts_with("avx512.mask.psra")) {
4584 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4585 bool IsVariable = Name[16] == 'v';
4586 char Size = Name[16] == '.' ? Name[17]
4587 : Name[17] == '.' ? Name[18]
4588 : Name[18] == '.' ? Name[19]
4589 : Name[20];
4590
4591 Intrinsic::ID IID;
4592 if (IsVariable && Name[17] != '.') {
4593 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4594 IID = Intrinsic::x86_avx2_psrav_d;
4595 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4596 IID = Intrinsic::x86_avx2_psrav_d_256;
4597 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4598 IID = Intrinsic::x86_avx512_psrav_w_128;
4599 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4600 IID = Intrinsic::x86_avx512_psrav_w_256;
4601 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4602 IID = Intrinsic::x86_avx512_psrav_w_512;
4603 else
4604 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4605 } else if (Name.ends_with(".128")) {
4606 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4607 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4608 : Intrinsic::x86_sse2_psra_d;
4609 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4610 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4611 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4612 : Intrinsic::x86_avx512_psra_q_128;
4613 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4614 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4615 : Intrinsic::x86_sse2_psra_w;
4616 else
4617 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4618 } else if (Name.ends_with(".256")) {
4619 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4620 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4621 : Intrinsic::x86_avx2_psra_d;
4622 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4623 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4624 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4625 : Intrinsic::x86_avx512_psra_q_256;
4626 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4627 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4628 : Intrinsic::x86_avx2_psra_w;
4629 else
4630 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4631 } else {
4632 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4633 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4634 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4635 : Intrinsic::x86_avx512_psra_d_512;
4636 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4637 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4638 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4639 : Intrinsic::x86_avx512_psra_q_512;
4640 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4641 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4642 : Intrinsic::x86_avx512_psra_w_512;
4643 else
4644 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4645 }
4646
4647 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4648 } else if (Name.starts_with("avx512.mask.move.s")) {
4649 Rep = upgradeMaskedMove(Builder, *CI);
4650 } else if (Name.starts_with("avx512.cvtmask2")) {
4651 Rep = upgradeMaskToInt(Builder, *CI);
4652 } else if (Name.ends_with(".movntdqa")) {
4653 MDNode *Node = MDNode::get(
4654 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4655
4656 LoadInst *LI = Builder.CreateAlignedLoad(
4657 CI->getType(), CI->getArgOperand(0),
4658 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4659 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4660 Rep = LI;
4661 } else if (Name.starts_with("fma.vfmadd.") ||
4662 Name.starts_with("fma.vfmsub.") ||
4663 Name.starts_with("fma.vfnmadd.") ||
4664 Name.starts_with("fma.vfnmsub.")) {
4665 bool NegMul = Name[6] == 'n';
4666 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4667 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4668
4669 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4670 CI->getArgOperand(2)};
4671
4672 if (IsScalar) {
4673 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4674 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4675 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4676 }
4677
4678 if (NegMul && !IsScalar)
4679 Ops[0] = Builder.CreateFNeg(Ops[0]);
4680 if (NegMul && IsScalar)
4681 Ops[1] = Builder.CreateFNeg(Ops[1]);
4682 if (NegAcc)
4683 Ops[2] = Builder.CreateFNeg(Ops[2]);
4684
4685 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4686
4687 if (IsScalar)
4688 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4689 } else if (Name.starts_with("fma4.vfmadd.s")) {
4690 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4691 CI->getArgOperand(2)};
4692
4693 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4694 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4695 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4696
4697 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4698
4699 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4700 Rep, (uint64_t)0);
4701 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4702 Name.starts_with("avx512.maskz.vfmadd.s") ||
4703 Name.starts_with("avx512.mask3.vfmadd.s") ||
4704 Name.starts_with("avx512.mask3.vfmsub.s") ||
4705 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4706 bool IsMask3 = Name[11] == '3';
4707 bool IsMaskZ = Name[11] == 'z';
4708 // Drop the "avx512.mask." to make it easier.
4709 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4710 bool NegMul = Name[2] == 'n';
4711 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4712
4713 Value *A = CI->getArgOperand(0);
4714 Value *B = CI->getArgOperand(1);
4715 Value *C = CI->getArgOperand(2);
4716
4717 if (NegMul && (IsMask3 || IsMaskZ))
4718 A = Builder.CreateFNeg(A);
4719 if (NegMul && !(IsMask3 || IsMaskZ))
4720 B = Builder.CreateFNeg(B);
4721 if (NegAcc)
4722 C = Builder.CreateFNeg(C);
4723
4724 A = Builder.CreateExtractElement(A, (uint64_t)0);
4725 B = Builder.CreateExtractElement(B, (uint64_t)0);
4726 C = Builder.CreateExtractElement(C, (uint64_t)0);
4727
4728 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4729 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4730 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4731
4732 Intrinsic::ID IID;
4733 if (Name.back() == 'd')
4734 IID = Intrinsic::x86_avx512_vfmadd_f64;
4735 else
4736 IID = Intrinsic::x86_avx512_vfmadd_f32;
4737 Rep = Builder.CreateIntrinsic(IID, Ops);
4738 } else {
4739 Rep = Builder.CreateFMA(A, B, C);
4740 }
4741
4742 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4743 : IsMask3 ? C
4744 : A;
4745
4746 // For Mask3 with NegAcc, we need to create a new extractelement that
4747 // avoids the negation above.
4748 if (NegAcc && IsMask3)
4749 PassThru =
4750 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4751
4752 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4753 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4754 (uint64_t)0);
4755 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4756 Name.starts_with("avx512.mask.vfnmadd.p") ||
4757 Name.starts_with("avx512.mask.vfnmsub.p") ||
4758 Name.starts_with("avx512.mask3.vfmadd.p") ||
4759 Name.starts_with("avx512.mask3.vfmsub.p") ||
4760 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4761 Name.starts_with("avx512.maskz.vfmadd.p")) {
4762 bool IsMask3 = Name[11] == '3';
4763 bool IsMaskZ = Name[11] == 'z';
4764 // Drop the "avx512.mask." to make it easier.
4765 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4766 bool NegMul = Name[2] == 'n';
4767 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4768
4769 Value *A = CI->getArgOperand(0);
4770 Value *B = CI->getArgOperand(1);
4771 Value *C = CI->getArgOperand(2);
4772
4773 if (NegMul && (IsMask3 || IsMaskZ))
4774 A = Builder.CreateFNeg(A);
4775 if (NegMul && !(IsMask3 || IsMaskZ))
4776 B = Builder.CreateFNeg(B);
4777 if (NegAcc)
4778 C = Builder.CreateFNeg(C);
4779
4780 if (CI->arg_size() == 5 &&
4781 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4782 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4783 Intrinsic::ID IID;
4784 // Check the character before ".512" in string.
4785 if (Name[Name.size() - 5] == 's')
4786 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4787 else
4788 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4789
4790 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4791 } else {
4792 Rep = Builder.CreateFMA(A, B, C);
4793 }
4794
4795 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4796 : IsMask3 ? CI->getArgOperand(2)
4797 : CI->getArgOperand(0);
4798
4799 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4800 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4801 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4802 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4803 Intrinsic::ID IID;
4804 if (VecWidth == 128 && EltWidth == 32)
4805 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4806 else if (VecWidth == 256 && EltWidth == 32)
4807 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4808 else if (VecWidth == 128 && EltWidth == 64)
4809 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4810 else if (VecWidth == 256 && EltWidth == 64)
4811 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4812 else
4813 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4814
4815 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4816 CI->getArgOperand(2)};
4817 Ops[2] = Builder.CreateFNeg(Ops[2]);
4818 Rep = Builder.CreateIntrinsic(IID, Ops);
4819 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4820 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4821 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4822 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4823 bool IsMask3 = Name[11] == '3';
4824 bool IsMaskZ = Name[11] == 'z';
4825 // Drop the "avx512.mask." to make it easier.
4826 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4827 bool IsSubAdd = Name[3] == 's';
4828 if (CI->arg_size() == 5) {
4829 Intrinsic::ID IID;
4830 // Check the character before ".512" in string.
4831 if (Name[Name.size() - 5] == 's')
4832 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4833 else
4834 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4835
4836 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4837 CI->getArgOperand(2), CI->getArgOperand(4)};
4838 if (IsSubAdd)
4839 Ops[2] = Builder.CreateFNeg(Ops[2]);
4840
4841 Rep = Builder.CreateIntrinsic(IID, Ops);
4842 } else {
4843 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4844
4845 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4846 CI->getArgOperand(2)};
4847
4848 Function *FMA = Intrinsic::getOrInsertDeclaration(
4849 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4850 Value *Odd = Builder.CreateCall(FMA, Ops);
4851 Ops[2] = Builder.CreateFNeg(Ops[2]);
4852 Value *Even = Builder.CreateCall(FMA, Ops);
4853
4854 if (IsSubAdd)
4855 std::swap(Even, Odd);
4856
4857 SmallVector<int, 32> Idxs(NumElts);
4858 for (int i = 0; i != NumElts; ++i)
4859 Idxs[i] = i + (i % 2) * NumElts;
4860
4861 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4862 }
4863
4864 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4865 : IsMask3 ? CI->getArgOperand(2)
4866 : CI->getArgOperand(0);
4867
4868 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4869 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4870 Name.starts_with("avx512.maskz.pternlog.")) {
4871 bool ZeroMask = Name[11] == 'z';
4872 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4873 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4874 Intrinsic::ID IID;
4875 if (VecWidth == 128 && EltWidth == 32)
4876 IID = Intrinsic::x86_avx512_pternlog_d_128;
4877 else if (VecWidth == 256 && EltWidth == 32)
4878 IID = Intrinsic::x86_avx512_pternlog_d_256;
4879 else if (VecWidth == 512 && EltWidth == 32)
4880 IID = Intrinsic::x86_avx512_pternlog_d_512;
4881 else if (VecWidth == 128 && EltWidth == 64)
4882 IID = Intrinsic::x86_avx512_pternlog_q_128;
4883 else if (VecWidth == 256 && EltWidth == 64)
4884 IID = Intrinsic::x86_avx512_pternlog_q_256;
4885 else if (VecWidth == 512 && EltWidth == 64)
4886 IID = Intrinsic::x86_avx512_pternlog_q_512;
4887 else
4888 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4889
4890 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4891 CI->getArgOperand(2), CI->getArgOperand(3)};
4892 Rep = Builder.CreateIntrinsic(IID, Args);
4893 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4894 : CI->getArgOperand(0);
4895 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4896 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4897 Name.starts_with("avx512.maskz.vpmadd52")) {
4898 bool ZeroMask = Name[11] == 'z';
4899 bool High = Name[20] == 'h' || Name[21] == 'h';
4900 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4901 Intrinsic::ID IID;
4902 if (VecWidth == 128 && !High)
4903 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4904 else if (VecWidth == 256 && !High)
4905 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4906 else if (VecWidth == 512 && !High)
4907 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4908 else if (VecWidth == 128 && High)
4909 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4910 else if (VecWidth == 256 && High)
4911 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4912 else if (VecWidth == 512 && High)
4913 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4914 else
4915 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4916
4917 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4918 CI->getArgOperand(2)};
4919 Rep = Builder.CreateIntrinsic(IID, Args);
4920 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4921 : CI->getArgOperand(0);
4922 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4923 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4924 Name.starts_with("avx512.mask.vpermt2var.") ||
4925 Name.starts_with("avx512.maskz.vpermt2var.")) {
4926 bool ZeroMask = Name[11] == 'z';
4927 bool IndexForm = Name[17] == 'i';
4928 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4929 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4930 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4931 Name.starts_with("avx512.mask.vpdpbusds.") ||
4932 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4933 bool ZeroMask = Name[11] == 'z';
4934 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4935 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4936 Intrinsic::ID IID;
4937 if (VecWidth == 128 && !IsSaturating)
4938 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4939 else if (VecWidth == 256 && !IsSaturating)
4940 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4941 else if (VecWidth == 512 && !IsSaturating)
4942 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4943 else if (VecWidth == 128 && IsSaturating)
4944 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4945 else if (VecWidth == 256 && IsSaturating)
4946 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4947 else if (VecWidth == 512 && IsSaturating)
4948 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4949 else
4950 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4951
4952 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4953 CI->getArgOperand(2)};
4954
4955 // Input arguments types were incorrectly set to vectors of i32 before but
4956 // they should be vectors of i8. Insert bit cast when encountering the old
4957 // types
4958 if (Args[1]->getType()->isVectorTy() &&
4959 cast<VectorType>(Args[1]->getType())
4960 ->getElementType()
4961 ->isIntegerTy(32) &&
4962 Args[2]->getType()->isVectorTy() &&
4963 cast<VectorType>(Args[2]->getType())
4964 ->getElementType()
4965 ->isIntegerTy(32)) {
4966 Type *NewArgType = nullptr;
4967 if (VecWidth == 128)
4968 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4969 else if (VecWidth == 256)
4970 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4971 else if (VecWidth == 512)
4972 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4973 else
4974 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4975 CI);
4976
4977 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4978 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4979 }
4980
4981 Rep = Builder.CreateIntrinsic(IID, Args);
4982 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4983 : CI->getArgOperand(0);
4984 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4985 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4986 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4987 Name.starts_with("avx512.mask.vpdpwssds.") ||
4988 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4989 bool ZeroMask = Name[11] == 'z';
4990 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4991 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4992 Intrinsic::ID IID;
4993 if (VecWidth == 128 && !IsSaturating)
4994 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4995 else if (VecWidth == 256 && !IsSaturating)
4996 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4997 else if (VecWidth == 512 && !IsSaturating)
4998 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4999 else if (VecWidth == 128 && IsSaturating)
5000 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5001 else if (VecWidth == 256 && IsSaturating)
5002 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5003 else if (VecWidth == 512 && IsSaturating)
5004 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5005 else
5006 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5007
5008 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5009 CI->getArgOperand(2)};
5010
5011 // Input arguments types were incorrectly set to vectors of i32 before but
5012 // they should be vectors of i16. Insert bit cast when encountering the old
5013 // types
5014 if (Args[1]->getType()->isVectorTy() &&
5015 cast<VectorType>(Args[1]->getType())
5016 ->getElementType()
5017 ->isIntegerTy(32) &&
5018 Args[2]->getType()->isVectorTy() &&
5019 cast<VectorType>(Args[2]->getType())
5020 ->getElementType()
5021 ->isIntegerTy(32)) {
5022 Type *NewArgType = nullptr;
5023 if (VecWidth == 128)
5024 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5025 else if (VecWidth == 256)
5026 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5027 else if (VecWidth == 512)
5028 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5029 else
5030 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5031 CI);
5032
5033 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5034 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5035 }
5036
5037 Rep = Builder.CreateIntrinsic(IID, Args);
5038 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5039 : CI->getArgOperand(0);
5040 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5041 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5042 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5043 Name == "subborrow.u32" || Name == "subborrow.u64") {
5044 Intrinsic::ID IID;
5045 if (Name[0] == 'a' && Name.back() == '2')
5046 IID = Intrinsic::x86_addcarry_32;
5047 else if (Name[0] == 'a' && Name.back() == '4')
5048 IID = Intrinsic::x86_addcarry_64;
5049 else if (Name[0] == 's' && Name.back() == '2')
5050 IID = Intrinsic::x86_subborrow_32;
5051 else if (Name[0] == 's' && Name.back() == '4')
5052 IID = Intrinsic::x86_subborrow_64;
5053 else
5054 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5055
5056 // Make a call with 3 operands.
5057 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5058 CI->getArgOperand(2)};
5059 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5060
5061 // Extract the second result and store it.
5062 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5063 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5064 // Replace the original call result with the first result of the new call.
5065 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5066
5067 CI->replaceAllUsesWith(CF);
5068 Rep = nullptr;
5069 } else if (Name.starts_with("avx512.mask.") &&
5070 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5071 // Rep will be updated by the call in the condition.
5072 } else if (Name.starts_with("bmi.pdep.")) {
5073 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5074 } else if (Name.starts_with("bmi.pext.")) {
5075 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5076 } else
5077 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5078
5079 return Rep;
5080}
5081
5082static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5083 Function *F, IRBuilder<> &Builder) {
5084 if (Name.starts_with("neon.bfcvt")) {
5085 if (Name.starts_with("neon.bfcvtn2")) {
5086 SmallVector<int, 32> LoMask(4);
5087 std::iota(LoMask.begin(), LoMask.end(), 0);
5088 SmallVector<int, 32> ConcatMask(8);
5089 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5090 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5091 Value *Trunc =
5092 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5093 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5094 } else if (Name.starts_with("neon.bfcvtn")) {
5095 SmallVector<int, 32> ConcatMask(8);
5096 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5097 Type *V4BF16 =
5098 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5099 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5100 dbgs() << "Trunc: " << *Trunc << "\n";
5101 return Builder.CreateShuffleVector(
5102 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5103 } else {
5104 return Builder.CreateFPTrunc(CI->getOperand(0),
5105 Type::getBFloatTy(F->getContext()));
5106 }
5107 } else if (Name.starts_with("sve.fcvt")) {
5108 Intrinsic::ID NewID =
5109 StringSwitch<Intrinsic::ID>(Name)
5110 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5111 .Case("sve.fcvtnt.bf16f32",
5112 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5113 .Default(Intrinsic::not_intrinsic);
5114 if (NewID == Intrinsic::not_intrinsic)
5115 llvm_unreachable("Unhandled Intrinsic!");
5116
5117 SmallVector<Value *, 3> Args(CI->args());
5118
5119 // The original intrinsics incorrectly used a predicate based on the
5120 // smallest element type rather than the largest.
5121 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5122 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5123
5124 if (Args[1]->getType() != BadPredTy)
5125 llvm_unreachable("Unexpected predicate type!");
5126
5127 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5128 BadPredTy, Args[1]);
5129 Args[1] = Builder.CreateIntrinsic(
5130 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5131
5132 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5133 CI->getName());
5134 }
5135
5136 if (Name == "neon.vcvtfp2hf")
5137 return Builder.CreateBitCast(
5138 Builder.CreateFPTrunc(
5139 CI->getOperand(0),
5140 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5141 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5142 if (Name == "neon.vcvthf2fp")
5143 return Builder.CreateFPExt(
5144 Builder.CreateBitCast(
5145 CI->getOperand(0),
5146 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5147 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5148
5149 llvm_unreachable("Unhandled Intrinsic!");
5150}
5151
5152static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5153 IRBuilder<> &Builder) {
5154 if (Name == "mve.vctp64.old") {
5155 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5156 // correct type.
5157 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5158 CI->getArgOperand(0),
5159 /*FMFSource=*/nullptr, CI->getName());
5160 Value *C1 = Builder.CreateIntrinsic(
5161 Intrinsic::arm_mve_pred_v2i,
5162 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5163 return Builder.CreateIntrinsic(
5164 Intrinsic::arm_mve_pred_i2v,
5165 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5166 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5167 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5168 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5169 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5170 Name ==
5171 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5172 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5173 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5174 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5175 Name ==
5176 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5177 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5178 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5179 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5180 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5181 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5182 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5183 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5184 std::vector<Type *> Tys;
5185 unsigned ID = CI->getIntrinsicID();
5186 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5187 switch (ID) {
5188 case Intrinsic::arm_mve_mull_int_predicated:
5189 case Intrinsic::arm_mve_vqdmull_predicated:
5190 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5191 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5192 break;
5193 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5194 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5195 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5196 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5197 V2I1Ty};
5198 break;
5199 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5200 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5201 CI->getOperand(1)->getType(), V2I1Ty};
5202 break;
5203 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5204 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5205 CI->getOperand(2)->getType(), V2I1Ty};
5206 break;
5207 case Intrinsic::arm_cde_vcx1q_predicated:
5208 case Intrinsic::arm_cde_vcx1qa_predicated:
5209 case Intrinsic::arm_cde_vcx2q_predicated:
5210 case Intrinsic::arm_cde_vcx2qa_predicated:
5211 case Intrinsic::arm_cde_vcx3q_predicated:
5212 case Intrinsic::arm_cde_vcx3qa_predicated:
5213 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5214 break;
5215 default:
5216 llvm_unreachable("Unhandled Intrinsic!");
5217 }
5218
5219 std::vector<Value *> Ops;
5220 for (Value *Op : CI->args()) {
5221 Type *Ty = Op->getType();
5222 if (Ty->getScalarSizeInBits() == 1) {
5223 Value *C1 = Builder.CreateIntrinsic(
5224 Intrinsic::arm_mve_pred_v2i,
5225 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5226 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5227 }
5228 Ops.push_back(Op);
5229 }
5230
5231 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5232 CI->getName());
5233 }
5234 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5235}
5236
5237// These are expected to have the arguments:
5238// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5239//
5240// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5241//
5242static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5243 Function *F, IRBuilder<> &Builder) {
5244 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5245 // for compatibility.
5246 auto UpgradeLegacyWMMAIUIntrinsicCall =
5247 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5248 ArrayRef<Type *> OverloadTys) -> Value * {
5249 // Prepare arguments, append clamp=0 for compatibility
5250 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5251 Args.push_back(Builder.getFalse());
5252
5253 // Insert the declaration for the right overload types
5254 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5255 F->getParent(), F->getIntrinsicID(), OverloadTys);
5256
5257 // Copy operand bundles if any
5258 SmallVector<OperandBundleDef, 1> Bundles;
5259 CI->getOperandBundlesAsDefs(Bundles);
5260
5261 // Create the new call and copy calling properties
5262 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5263 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5264 NewCall->setCallingConv(CI->getCallingConv());
5265 NewCall->setAttributes(CI->getAttributes());
5266 NewCall->copyMetadata(*CI);
5267 return NewCall;
5268 };
5269
5270 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5271 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5272 "intrinsic should have 7 arguments");
5273 Type *T1 = CI->getArgOperand(4)->getType();
5274 Type *T2 = CI->getArgOperand(1)->getType();
5275 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5276 }
5277 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5278 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5279 "intrinsic should have 8 arguments");
5280 Type *T1 = CI->getArgOperand(4)->getType();
5281 Type *T2 = CI->getArgOperand(1)->getType();
5282 Type *T3 = CI->getArgOperand(3)->getType();
5283 Type *T4 = CI->getArgOperand(5)->getType();
5284 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5285 }
5286
5287 switch (F->getIntrinsicID()) {
5288 default:
5289 break;
5290 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5291 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5292 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5293 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5294 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5295 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5296 // Drop src0 and src1 modifiers.
5297 const Value *Op0 = CI->getArgOperand(0);
5298 const Value *Op2 = CI->getArgOperand(2);
5299 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5300 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5301 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5302 if (!ModA->isZero() || !ModB->isZero())
5303 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5304
5305 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5306 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5307 Args.push_back(CI->getArgOperand(I));
5308
5309 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5310 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5311 Overloads.push_back(Args[3]->getType());
5312 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5313 F->getParent(), F->getIntrinsicID(), Overloads);
5314
5315 SmallVector<OperandBundleDef, 1> Bundles;
5316 CI->getOperandBundlesAsDefs(Bundles);
5317
5318 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5319 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5320 NewCall->setCallingConv(CI->getCallingConv());
5321 NewCall->setAttributes(CI->getAttributes());
5322 NewCall->copyMetadata(*CI);
5323 NewCall->takeName(CI);
5324 return NewCall;
5325 }
5326 }
5327
5328 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5329 Value *LHS = CI->getArgOperand(0);
5330 Value *RHS = CI->getArgOperand(1);
5331 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5332 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5333 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5334 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5335 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5336 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5337 NewCall->setCallingConv(CI->getCallingConv());
5338 NewCall->copyMetadata(*CI);
5339 NewCall->takeName(CI);
5340 return NewCall;
5341 }
5342
5343 if (Name.starts_with("addrspacecast.nonnull")) {
5344 if (CI->getNumOperands() < 2) // Malformed bitcode.
5345 return nullptr;
5346 Value *ASC = Builder.CreateAddrSpaceCast(
5347 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5348 ASC->takeName(CI);
5349 return ASC;
5350 }
5351
5352 AtomicRMWInst::BinOp RMWOp =
5353 StringSwitch<AtomicRMWInst::BinOp>(Name)
5354 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5355 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5356 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5357 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5358 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5359 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5360 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5361 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5362 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5363 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5364 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5365 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5366 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5367
5368 unsigned NumOperands = CI->getNumOperands();
5369 if (NumOperands < 3) // Malformed bitcode.
5370 return nullptr;
5371
5372 Value *Ptr = CI->getArgOperand(0);
5373 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5374 if (!PtrTy) // Malformed.
5375 return nullptr;
5376
5377 Value *Val = CI->getArgOperand(1);
5378 if (Val->getType() != CI->getType()) // Malformed.
5379 return nullptr;
5380
5381 ConstantInt *OrderArg = nullptr;
5382 bool IsVolatile = false;
5383
5384 // These should have 5 arguments (plus the callee). A separate version of the
5385 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5386 if (NumOperands > 3)
5387 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5388
5389 // Ignore scope argument at 3
5390
5391 if (NumOperands > 5) {
5392 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5393 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5394 }
5395
5396 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5397 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5398 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5399 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5400 Order = AtomicOrdering::SequentiallyConsistent;
5401
5402 LLVMContext &Ctx = F->getContext();
5403
5404 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5405 Type *RetTy = CI->getType();
5406 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5407 if (VT->getElementType()->isIntegerTy(16)) {
5408 VectorType *AsBF16 =
5409 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5410 Val = Builder.CreateBitCast(Val, AsBF16);
5411 }
5412 }
5413
5414 // The scope argument never really worked correctly. Use agent as the most
5415 // conservative option which should still always produce the instruction.
5416 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5417 AtomicRMWInst *RMW =
5418 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5419
5420 unsigned AddrSpace = PtrTy->getAddressSpace();
5421 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5422 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5423 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5424 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5425 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5426 }
5427
5428 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5429 MDBuilder MDB(F->getContext());
5430 MDNode *RangeNotPrivate =
5431 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5432 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5433 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5434 }
5435
5436 if (IsVolatile)
5437 RMW->setVolatile(true);
5438
5439 return Builder.CreateBitCast(RMW, RetTy);
5440}
5441
5442/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5443/// plain MDNode, as it's the verifier's job to check these are the correct
5444/// types later.
5445static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5446 if (Op < CI->arg_size()) {
5447 if (MetadataAsValue *MAV =
5448 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5449 Metadata *MD = MAV->getMetadata();
5450 return dyn_cast_if_present<MDNode>(MD);
5451 }
5452 }
5453 return nullptr;
5454}
5455
5456/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5457static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5458 if (Op < CI->arg_size())
5459 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5460 return MAV->getMetadata();
5461 return nullptr;
5462}
5463
5464/// Convert debug intrinsic calls to non-instruction debug records.
5465/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5466/// \p CI - The debug intrinsic call.
5467static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5468 DbgRecord *DR = nullptr;
5469 if (Name == "label") {
5470 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5471 } else if (Name == "assign") {
5472 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5473 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5474 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5475 unwrapMAVMetadataOp(CI, 4),
5476 /*The address is a Value ref, it will be stored as a Metadata */
5477 unwrapMAVOp(CI, 5));
5478 } else if (Name == "declare") {
5479 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5480 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5481 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5482 } else if (Name == "addr") {
5483 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5484 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5485 // Don't try to add something to the expression if it's not an expression.
5486 // Instead, allow the verifier to fail later.
5487 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5488 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5489 }
5490 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5491 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5492 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5493 } else if (Name == "value") {
5494 // An old version of dbg.value had an extra offset argument.
5495 unsigned VarOp = 1;
5496 unsigned ExprOp = 2;
5497 if (CI->arg_size() == 4) {
5498 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5499 // Nonzero offset dbg.values get dropped without a replacement.
5500 if (!Offset || !Offset->isNullValue())
5501 return;
5502 VarOp = 2;
5503 ExprOp = 3;
5504 }
5505 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5506 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5507 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5508 nullptr);
5509 }
5510 DR->setDebugLoc(CI->getDebugLoc());
5511 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5512 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5513}
5514
5515static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5516 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5517 if (!Offset)
5518 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5519 int64_t OffsetVal = Offset->getSExtValue();
5520 return Builder.CreateIntrinsic(OffsetVal >= 0
5521 ? Intrinsic::vector_splice_left
5522 : Intrinsic::vector_splice_right,
5523 CI->getType(),
5524 {CI->getArgOperand(0), CI->getArgOperand(1),
5525 Builder.getInt32(std::abs(OffsetVal))});
5526}
5527
5528static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5529 Function *F, IRBuilder<> &Builder) {
5530 if (Name.starts_with("to.fp16")) {
5531 Value *Cast =
5532 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5533 return Builder.CreateBitCast(Cast, CI->getType());
5534 }
5535
5536 if (Name.starts_with("from.fp16")) {
5537 Value *Cast =
5538 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5539 return Builder.CreateFPExt(Cast, CI->getType());
5540 }
5541
5542 return nullptr;
5543}
5544
5545static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5546 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5547 if (!MD || !isa<MDString>(MD))
5548 return ICmpInst::BAD_ICMP_PREDICATE;
5549 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5550 .Case("eq", ICmpInst::ICMP_EQ)
5551 .Case("ne", ICmpInst::ICMP_NE)
5552 .Case("ugt", ICmpInst::ICMP_UGT)
5553 .Case("uge", ICmpInst::ICMP_UGE)
5554 .Case("ult", ICmpInst::ICMP_ULT)
5555 .Case("ule", ICmpInst::ICMP_ULE)
5556 .Case("sgt", ICmpInst::ICMP_SGT)
5557 .Case("sge", ICmpInst::ICMP_SGE)
5558 .Case("slt", ICmpInst::ICMP_SLT)
5559 .Case("sle", ICmpInst::ICMP_SLE)
5560 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5561}
5562
5563static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5564 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5565 if (!MD || !isa<MDString>(MD))
5566 return FCmpInst::BAD_FCMP_PREDICATE;
5567 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5568 .Case("oeq", FCmpInst::FCMP_OEQ)
5569 .Case("ogt", FCmpInst::FCMP_OGT)
5570 .Case("oge", FCmpInst::FCMP_OGE)
5571 .Case("olt", FCmpInst::FCMP_OLT)
5572 .Case("ole", FCmpInst::FCMP_OLE)
5573 .Case("one", FCmpInst::FCMP_ONE)
5574 .Case("ord", FCmpInst::FCMP_ORD)
5575 .Case("uno", FCmpInst::FCMP_UNO)
5576 .Case("ueq", FCmpInst::FCMP_UEQ)
5577 .Case("ugt", FCmpInst::FCMP_UGT)
5578 .Case("uge", FCmpInst::FCMP_UGE)
5579 .Case("ult", FCmpInst::FCMP_ULT)
5580 .Case("ule", FCmpInst::FCMP_ULE)
5581 .Case("une", FCmpInst::FCMP_UNE)
5582 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5583}
5584
5585static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5586 IRBuilder<> &Builder) {
5587 Value *Rep;
5588 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5589 if (Opcode && Instruction::isUnaryOp(Opcode))
5590 Rep =
5591 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5592 else if (Opcode && Instruction::isBinaryOp(Opcode))
5593 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5594 CI->getArgOperand(0), CI->getArgOperand(1));
5595 else if (Opcode && Instruction::isCast(Opcode))
5596 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5597 CI->getType());
5598 else if (Opcode == Instruction::ICmp)
5599 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5600 CI->getArgOperand(0), CI->getArgOperand(1));
5601 else if (Opcode == Instruction::FCmp)
5602 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5603 CI->getArgOperand(0), CI->getArgOperand(1));
5604 else if (Opcode == Instruction::Select)
5605 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5606 CI->getArgOperand(2));
5607 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5608 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5609 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5610 } else
5611 llvm_unreachable("Unexpected vp intrinsic");
5612 Rep->takeName(CI);
5613 return Rep;
5614}
5615
5616static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5617 IRBuilder<> &Builder) {
5618 Intrinsic::ID IID = NewFn->getIntrinsicID();
5619
5620 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5621 if (Defaults.empty())
5622 return false;
5623
5624 unsigned OldArgCount = CI->arg_size();
5625 unsigned NewArgCount = NewFn->arg_size();
5626
5627 if (OldArgCount < FirstDefault)
5628 return false;
5629
5630 // More arguments than the new intrinsic accepts, cannot upgrade.
5631 if (OldArgCount > NewArgCount)
5632 return false;
5633
5634 // The call already passes the defaulted arguments explicitly; only the
5635 // callee is still the old, shorter declaration, so retarget it.
5636 if (OldArgCount == NewArgCount) {
5637 if (CI->getFunctionType() != NewFn->getFunctionType())
5638 return false;
5639 CI->setCalledFunction(NewFn);
5640 return true;
5641 }
5642
5643 // OldArgCount < NewArgCount: Fill in each missing trailing default
5644 // argument from the table.
5645 SmallVector<Value *, 8> NewArgs(CI->args());
5646
5647 FunctionType *NewFT = NewFn->getFunctionType();
5648 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5649 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5650 "missing argument outside the default range");
5651 Type *ParamTy = NewFT->getParamType(Idx);
5652
5653 // Only integer types are supported (i1, i8, i16, i32, i64).
5654 if (!ParamTy->isIntegerTy())
5655 return false;
5656 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5657 }
5658
5659 // Preserve operand bundles by creating the call with them.
5660 SmallVector<OperandBundleDef, 1> OpBundles;
5661 CI->getOperandBundlesAsDefs(OpBundles);
5662 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5663
5664 NewCall->takeName(CI);
5665 NewCall->setCallingConv(CI->getCallingConv());
5666 NewCall->copyMetadata(*CI);
5667 if (auto *OldCI = dyn_cast<CallInst>(CI))
5668 NewCall->setTailCallKind(OldCI->getTailCallKind());
5669
5670 CI->replaceAllUsesWith(NewCall);
5671 CI->eraseFromParent();
5672 return true;
5673}
5674
5675/// Upgrade a call to an old intrinsic. All argument and return casting must be
5676/// provided to seamlessly integrate with existing context.
5677void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5678 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5679 // checks the callee's function type matches. It's likely we need to handle
5680 // type changes here.
5681 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5682 if (!F)
5683 return;
5684
5685 LLVMContext &C = CI->getContext();
5686 IRBuilder<> Builder(C);
5687 if (isa<FPMathOperator>(CI))
5688 Builder.setFastMathFlags(CI->getFastMathFlags());
5689 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5690
5691 if (!NewFn) {
5692 // Get the Function's name.
5693 StringRef Name = F->getName();
5694 if (!Name.consume_front("llvm."))
5695 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5696
5697 bool IsX86 = Name.consume_front("x86.");
5698 bool IsNVVM = Name.consume_front("nvvm.");
5699 bool IsAArch64 = Name.consume_front("aarch64.");
5700 bool IsARM = Name.consume_front("arm.");
5701 bool IsAMDGCN = Name.consume_front("amdgcn.");
5702 bool IsDbg = Name.consume_front("dbg.");
5703 bool IsOldSplice =
5704 (Name.consume_front("experimental.vector.splice") ||
5705 Name.consume_front("vector.splice")) &&
5706 !(Name.starts_with(".left") || Name.starts_with(".right"));
5707 Value *Rep = nullptr;
5708
5709 if (!IsX86 && Name == "stackprotectorcheck") {
5710 Rep = nullptr;
5711 } else if (IsNVVM) {
5712 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5713 } else if (IsX86) {
5714 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5715 } else if (IsAArch64) {
5716 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5717 } else if (IsARM) {
5718 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5719 } else if (IsAMDGCN) {
5720 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5721 } else if (IsDbg) {
5722 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5723 } else if (IsOldSplice) {
5724 Rep = upgradeVectorSplice(CI, Builder);
5725 } else if (Name.consume_front("convert.")) {
5726 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5727 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5728 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5729 Rep = nullptr;
5730 } else if (shouldUpgradeVPIntrinsic(Name)) {
5731 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5732 } else {
5733 llvm_unreachable("Unknown function for CallBase upgrade.");
5734 }
5735
5736 if (Rep)
5737 CI->replaceAllUsesWith(Rep);
5738 CI->eraseFromParent();
5739 return;
5740 }
5741
5742 const auto &DefaultCase = [&]() -> void {
5743 if (F == NewFn)
5744 return;
5745
5746 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5747 // Handle generic mangling change.
5748 assert(
5749 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5750 "Unknown function for CallBase upgrade and isn't just a name change");
5751 CI->setCalledFunction(NewFn);
5752 return;
5753 }
5754
5755 // This must be an upgrade from a named to a literal struct.
5756 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5757 assert(OldST != NewFn->getReturnType() &&
5758 "Return type must have changed");
5759 assert(OldST->getNumElements() ==
5760 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5761 "Must have same number of elements");
5762
5763 SmallVector<Value *> Args(CI->args());
5764 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5765 NewCI->setAttributes(CI->getAttributes());
5766 Value *Res = PoisonValue::get(OldST);
5767 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5768 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5769 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5770 }
5771 CI->replaceAllUsesWith(Res);
5772 CI->eraseFromParent();
5773 return;
5774 }
5775
5776 // We're probably about to produce something invalid. Let the verifier catch
5777 // it instead of dying here.
5778 CI->setCalledOperand(
5779 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5780 return;
5781 };
5782 CallInst *NewCall = nullptr;
5783 switch (NewFn->getIntrinsicID()) {
5784 default: {
5785 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5786 return;
5787 DefaultCase();
5788 return;
5789 }
5790 case Intrinsic::arm_neon_vst1:
5791 case Intrinsic::arm_neon_vst2:
5792 case Intrinsic::arm_neon_vst3:
5793 case Intrinsic::arm_neon_vst4:
5794 case Intrinsic::arm_neon_vst2lane:
5795 case Intrinsic::arm_neon_vst3lane:
5796 case Intrinsic::arm_neon_vst4lane: {
5797 SmallVector<Value *, 4> Args(CI->args());
5798 NewCall = Builder.CreateCall(NewFn, Args);
5799 break;
5800 }
5801 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5802 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5803 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5804 LLVMContext &Ctx = F->getParent()->getContext();
5805 SmallVector<Value *, 4> Args(CI->args());
5806 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5807 cast<ConstantInt>(Args[3])->getZExtValue());
5808 NewCall = Builder.CreateCall(NewFn, Args);
5809 break;
5810 }
5811 case Intrinsic::aarch64_sve_ld3_sret:
5812 case Intrinsic::aarch64_sve_ld4_sret:
5813 case Intrinsic::aarch64_sve_ld2_sret: {
5814 // Is this a trivial remangle of the name to support ptr address spaces?
5815 if (isa<StructType>(F->getReturnType())) {
5816 DefaultCase();
5817 return;
5818 }
5819
5820 StringRef Name = F->getName();
5821 Name = Name.substr(5);
5822 unsigned N = StringSwitch<unsigned>(Name)
5823 .StartsWith("aarch64.sve.ld2", 2)
5824 .StartsWith("aarch64.sve.ld3", 3)
5825 .StartsWith("aarch64.sve.ld4", 4)
5826 .Default(0);
5827 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5828 unsigned MinElts = RetTy->getMinNumElements() / N;
5829 SmallVector<Value *, 2> Args(CI->args());
5830 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5831 Value *Ret = llvm::PoisonValue::get(RetTy);
5832 for (unsigned I = 0; I < N; I++) {
5833 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5834 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5835 }
5836 NewCall = dyn_cast<CallInst>(Ret);
5837 break;
5838 }
5839
5840 case Intrinsic::coro_end_async:
5841 case Intrinsic::coro_end: {
5842 SmallVector<Value *, 3> Args(CI->args());
5843 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5844 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5845 NewCall = Builder.CreateCall(NewFn, Args);
5846
5847 if (!CI->getType()->isVoidTy()) {
5848 if (!CI->use_empty()) {
5849 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5850 CI->getModule(), Intrinsic::coro_is_in_ramp);
5851 Value *InRamp = Builder.CreateCall(IsInRamp);
5852 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5853 }
5854 CI->eraseFromParent();
5855 return;
5856 }
5857
5858 break;
5859 }
5860
5861 case Intrinsic::vector_extract: {
5862 StringRef Name = F->getName();
5863 Name = Name.substr(5); // Strip llvm
5864 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5865 DefaultCase();
5866 return;
5867 }
5868 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5869 unsigned MinElts = RetTy->getMinNumElements();
5870 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5871 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5872 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5873 break;
5874 }
5875
5876 case Intrinsic::vector_insert: {
5877 StringRef Name = F->getName();
5878 Name = Name.substr(5);
5879 if (!Name.starts_with("aarch64.sve.tuple")) {
5880 DefaultCase();
5881 return;
5882 }
5883 if (Name.starts_with("aarch64.sve.tuple.set")) {
5884 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5885 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5886 Value *NewIdx =
5887 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5888 NewCall = Builder.CreateCall(
5889 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5890 break;
5891 }
5892 if (Name.starts_with("aarch64.sve.tuple.create")) {
5893 unsigned N = StringSwitch<unsigned>(Name)
5894 .StartsWith("aarch64.sve.tuple.create2", 2)
5895 .StartsWith("aarch64.sve.tuple.create3", 3)
5896 .StartsWith("aarch64.sve.tuple.create4", 4)
5897 .Default(0);
5898 assert(N > 1 && "Create is expected to be between 2-4");
5899 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5900 Value *Ret = llvm::PoisonValue::get(RetTy);
5901 unsigned MinElts = RetTy->getMinNumElements() / N;
5902 for (unsigned I = 0; I < N; I++) {
5903 Value *V = CI->getArgOperand(I);
5904 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5905 }
5906 NewCall = dyn_cast<CallInst>(Ret);
5907 }
5908 break;
5909 }
5910
5911 case Intrinsic::arm_neon_bfdot:
5912 case Intrinsic::arm_neon_bfmmla:
5913 case Intrinsic::arm_neon_bfmlalb:
5914 case Intrinsic::arm_neon_bfmlalt:
5915 case Intrinsic::aarch64_neon_bfdot:
5916 case Intrinsic::aarch64_neon_bfmmla:
5917 case Intrinsic::aarch64_neon_bfmlalb:
5918 case Intrinsic::aarch64_neon_bfmlalt: {
5919 SmallVector<Value *, 3> Args;
5920 assert(CI->arg_size() == 3 &&
5921 "Mismatch between function args and call args");
5922 size_t OperandWidth =
5923 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5924 assert((OperandWidth == 64 || OperandWidth == 128) &&
5925 "Unexpected operand width");
5926 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5927 auto Iter = CI->args().begin();
5928 Args.push_back(*Iter++);
5929 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5930 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5931 NewCall = Builder.CreateCall(NewFn, Args);
5932 break;
5933 }
5934
5935 case Intrinsic::bitreverse:
5936 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5937 break;
5938
5939 case Intrinsic::ctlz:
5940 case Intrinsic::cttz: {
5941 if (CI->arg_size() != 1) {
5942 DefaultCase();
5943 return;
5944 }
5945
5946 NewCall =
5947 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5948 break;
5949 }
5950
5951 case Intrinsic::objectsize: {
5952 Value *NullIsUnknownSize =
5953 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5954 Value *Dynamic =
5955 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5956 NewCall = Builder.CreateCall(
5957 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5958 break;
5959 }
5960
5961 case Intrinsic::ctpop:
5962 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5963 break;
5964 case Intrinsic::dbg_value: {
5965 StringRef Name = F->getName();
5966 Name = Name.substr(5); // Strip llvm.
5967 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5968 if (Name.starts_with("dbg.addr")) {
5969 DIExpression *Expr = cast<DIExpression>(
5970 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5971 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5972 NewCall =
5973 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5974 MetadataAsValue::get(C, Expr)});
5975 break;
5976 }
5977
5978 // Upgrade from the old version that had an extra offset argument.
5979 assert(CI->arg_size() == 4);
5980 // Drop nonzero offsets instead of attempting to upgrade them.
5981 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5982 if (Offset->isNullValue()) {
5983 NewCall = Builder.CreateCall(
5984 NewFn,
5985 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5986 break;
5987 }
5988 CI->eraseFromParent();
5989 return;
5990 }
5991
5992 case Intrinsic::ptr_annotation:
5993 // Upgrade from versions that lacked the annotation attribute argument.
5994 if (CI->arg_size() != 4) {
5995 DefaultCase();
5996 return;
5997 }
5998
5999 // Create a new call with an added null annotation attribute argument.
6000 NewCall = Builder.CreateCall(
6001 NewFn,
6002 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6003 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6004 NewCall->takeName(CI);
6005 CI->replaceAllUsesWith(NewCall);
6006 CI->eraseFromParent();
6007 return;
6008
6009 case Intrinsic::var_annotation:
6010 // Upgrade from versions that lacked the annotation attribute argument.
6011 if (CI->arg_size() != 4) {
6012 DefaultCase();
6013 return;
6014 }
6015 // Create a new call with an added null annotation attribute argument.
6016 NewCall = Builder.CreateCall(
6017 NewFn,
6018 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6019 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6020 NewCall->takeName(CI);
6021 CI->replaceAllUsesWith(NewCall);
6022 CI->eraseFromParent();
6023 return;
6024
6025 case Intrinsic::riscv_aes32dsi:
6026 case Intrinsic::riscv_aes32dsmi:
6027 case Intrinsic::riscv_aes32esi:
6028 case Intrinsic::riscv_aes32esmi:
6029 case Intrinsic::riscv_sm4ks:
6030 case Intrinsic::riscv_sm4ed: {
6031 // The last argument to these intrinsics used to be i8 and changed to i32.
6032 // The type overload for sm4ks and sm4ed was removed.
6033 Value *Arg2 = CI->getArgOperand(2);
6034 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6035 return;
6036
6037 Value *Arg0 = CI->getArgOperand(0);
6038 Value *Arg1 = CI->getArgOperand(1);
6039 if (CI->getType()->isIntegerTy(64)) {
6040 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6041 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6042 }
6043
6044 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6045 cast<ConstantInt>(Arg2)->getZExtValue());
6046
6047 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6048 Value *Res = NewCall;
6049 if (Res->getType() != CI->getType())
6050 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6051 NewCall->takeName(CI);
6052 CI->replaceAllUsesWith(Res);
6053 CI->eraseFromParent();
6054 return;
6055 }
6056 case Intrinsic::nvvm_mapa_shared_cluster: {
6057 // Create a new call with the correct address space.
6058 NewCall =
6059 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6060 Value *Res = NewCall;
6061 Res = Builder.CreateAddrSpaceCast(
6062 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6063 NewCall->takeName(CI);
6064 CI->replaceAllUsesWith(Res);
6065 CI->eraseFromParent();
6066 return;
6067 }
6068 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
6069 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6070 // Create a new call with the correct address space.
6071 SmallVector<Value *, 4> Args(CI->args());
6072 Args[0] = Builder.CreateAddrSpaceCast(
6073 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6074
6075 NewCall = Builder.CreateCall(NewFn, Args);
6076 NewCall->takeName(CI);
6077 CI->replaceAllUsesWith(NewCall);
6078 CI->eraseFromParent();
6079 return;
6080 }
6081 // clang-format off
6082#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6083 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6084 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6085#undef G2S_CLUSTER_CASE
6086 {
6087 SmallVector<Value *, 16> Args(CI->args());
6088 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6089 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6090 Args[0] = Builder.CreateAddrSpaceCast(
6091 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6092
6093 // Append the missing trailing arguments with default values (cta_group,
6094 // validate_pattern).
6095 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6096 Args.push_back(Builder.getInt32(0));
6097
6098 NewCall = Builder.CreateCall(NewFn, Args);
6099 NewCall->takeName(CI);
6100 CI->replaceAllUsesWith(NewCall);
6101 CI->eraseFromParent();
6102 return;
6103 }
6104
6105#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6106 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6107 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6108#undef G2S_CTA_CASE
6109 {
6110 SmallVector<Value *, 16> Args(CI->args());
6111 // Append the missing trailing validate_pattern argument with default
6112 // value 0.
6113 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6114 "expected only the trailing validate_pattern to be missing");
6115 Args.push_back(Builder.getInt32(0));
6116
6117 NewCall = Builder.CreateCall(NewFn, Args);
6118 NewCall->takeName(CI);
6119 CI->replaceAllUsesWith(NewCall);
6120 CI->eraseFromParent();
6121 return;
6122 }
6123#undef NVVM_TMA_G2S_MODES
6124 // clang-format on
6125
6126 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6127 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6128 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6129 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6130 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6131 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6132 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6133 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6134 StringRef Name = F->getName();
6135 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6136 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6137
6138 SmallVector<Value *, 16> Args(CI->args());
6139 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6140 NewCall = Builder.CreateCall(NewFn, Args);
6141 break;
6142 }
6143 case Intrinsic::nvvm_tcgen05_mma_shared:
6144 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
6145 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
6146 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
6147 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
6148 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
6149 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
6150 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
6151 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
6152 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
6153 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
6154 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
6155 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
6156 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
6157 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
6158 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
6159 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
6160 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
6161 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
6162 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
6163 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
6164 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
6165 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
6166 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
6167 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
6168 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
6169 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
6170 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
6171 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
6172 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
6173 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
6174 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
6175 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
6176 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
6177 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
6178 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
6179 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
6180 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
6181 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
6182 case Intrinsic::
6183 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
6184 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
6185 case Intrinsic::
6186 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
6187 case Intrinsic::nvvm_tcgen05_mma_tensor:
6188 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
6189 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
6190 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
6191 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
6192 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
6193 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
6194 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
6195 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
6196 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
6197 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
6198 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
6199 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
6200 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
6201 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
6202 case Intrinsic::
6203 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
6204 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
6205 case Intrinsic::
6206 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
6207 SmallVector<Value *, 12> Args(CI->args());
6208 Args.push_back(Builder.getInt32(0)); // collector_usage_b = discard(0)
6209 NewCall = Builder.CreateCall(NewFn, Args);
6210 break;
6211 }
6212 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6213 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6214 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6215 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6216 NewCall =
6217 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6218 Builder.getFalse()});
6219 break;
6220 case Intrinsic::nvvm_mbarrier_init: {
6221 SmallVector<Value *, 3> Args(CI->args());
6222 // The .shared variant folded into the overloaded form without gaining an
6223 // operand, so only the pre-layout two-argument form needs one appended.
6224 if (Args.size() == 2)
6225 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6226 NewCall = Builder.CreateCall(NewFn, Args);
6227 break;
6228 }
6229 case Intrinsic::riscv_sha256sig0:
6230 case Intrinsic::riscv_sha256sig1:
6231 case Intrinsic::riscv_sha256sum0:
6232 case Intrinsic::riscv_sha256sum1:
6233 case Intrinsic::riscv_sm3p0:
6234 case Intrinsic::riscv_sm3p1: {
6235 // The last argument to these intrinsics used to be i8 and changed to i32.
6236 // The type overload for sm4ks and sm4ed was removed.
6237 if (!CI->getType()->isIntegerTy(64))
6238 return;
6239
6240 Value *Arg =
6241 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6242
6243 NewCall = Builder.CreateCall(NewFn, Arg);
6244 Value *Res =
6245 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6246 NewCall->takeName(CI);
6247 CI->replaceAllUsesWith(Res);
6248 CI->eraseFromParent();
6249 return;
6250 }
6251
6252 case Intrinsic::x86_xop_vfrcz_ss:
6253 case Intrinsic::x86_xop_vfrcz_sd:
6254 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6255 break;
6256
6257 case Intrinsic::x86_xop_vpermil2pd:
6258 case Intrinsic::x86_xop_vpermil2ps:
6259 case Intrinsic::x86_xop_vpermil2pd_256:
6260 case Intrinsic::x86_xop_vpermil2ps_256: {
6261 SmallVector<Value *, 4> Args(CI->args());
6262 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6263 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6264 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6265 NewCall = Builder.CreateCall(NewFn, Args);
6266 break;
6267 }
6268
6269 case Intrinsic::x86_sse41_ptestc:
6270 case Intrinsic::x86_sse41_ptestz:
6271 case Intrinsic::x86_sse41_ptestnzc: {
6272 // The arguments for these intrinsics used to be v4f32, and changed
6273 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6274 // So, the only thing required is a bitcast for both arguments.
6275 // First, check the arguments have the old type.
6276 Value *Arg0 = CI->getArgOperand(0);
6277 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6278 return;
6279
6280 // Old intrinsic, add bitcasts
6281 Value *Arg1 = CI->getArgOperand(1);
6282
6283 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6284
6285 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6286 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6287
6288 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6289 break;
6290 }
6291
6292 case Intrinsic::x86_rdtscp: {
6293 // This used to take 1 arguments. If we have no arguments, it is already
6294 // upgraded.
6295 if (CI->getNumOperands() == 0)
6296 return;
6297
6298 NewCall = Builder.CreateCall(NewFn);
6299 // Extract the second result and store it.
6300 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6301 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6302 // Replace the original call result with the first result of the new call.
6303 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6304
6305 NewCall->takeName(CI);
6306 CI->replaceAllUsesWith(TSC);
6307 CI->eraseFromParent();
6308 return;
6309 }
6310
6311 case Intrinsic::x86_sse41_insertps:
6312 case Intrinsic::x86_sse41_dppd:
6313 case Intrinsic::x86_sse41_dpps:
6314 case Intrinsic::x86_sse41_mpsadbw:
6315 case Intrinsic::x86_avx_dp_ps_256:
6316 case Intrinsic::x86_avx2_mpsadbw: {
6317 // Need to truncate the last argument from i32 to i8 -- this argument models
6318 // an inherently 8-bit immediate operand to these x86 instructions.
6319 SmallVector<Value *, 4> Args(CI->args());
6320
6321 // Replace the last argument with a trunc.
6322 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6323 NewCall = Builder.CreateCall(NewFn, Args);
6324 break;
6325 }
6326
6327 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6328 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6329 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6330 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6331 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6332 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6333 SmallVector<Value *, 4> Args(CI->args());
6334 unsigned NumElts =
6335 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6336 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6337
6338 NewCall = Builder.CreateCall(NewFn, Args);
6339 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6340
6341 NewCall->takeName(CI);
6342 CI->replaceAllUsesWith(Res);
6343 CI->eraseFromParent();
6344 return;
6345 }
6346
6347 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6348 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6349 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6350 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6351 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6352 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6353 SmallVector<Value *, 4> Args(CI->args());
6354 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6355 if (NewFn->getIntrinsicID() ==
6356 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6357 Args[1] = Builder.CreateBitCast(
6358 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6359
6360 NewCall = Builder.CreateCall(NewFn, Args);
6361 Value *Res = Builder.CreateBitCast(
6362 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6363
6364 NewCall->takeName(CI);
6365 CI->replaceAllUsesWith(Res);
6366 CI->eraseFromParent();
6367 return;
6368 }
6369 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6370 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6371 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6372 SmallVector<Value *, 4> Args(CI->args());
6373 unsigned NumElts =
6374 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6375 Args[1] = Builder.CreateBitCast(
6376 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6377 Args[2] = Builder.CreateBitCast(
6378 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6379
6380 NewCall = Builder.CreateCall(NewFn, Args);
6381 break;
6382 }
6383
6384 case Intrinsic::thread_pointer: {
6385 NewCall = Builder.CreateCall(NewFn, {});
6386 break;
6387 }
6388
6389 case Intrinsic::memcpy:
6390 case Intrinsic::memmove:
6391 case Intrinsic::memset: {
6392 // We have to make sure that the call signature is what we're expecting.
6393 // We only want to change the old signatures by removing the alignment arg:
6394 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6395 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6396 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6397 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6398 // Note: i8*'s in the above can be any pointer type
6399 if (CI->arg_size() != 5) {
6400 DefaultCase();
6401 return;
6402 }
6403 // Remove alignment argument (3), and add alignment attributes to the
6404 // dest/src pointers.
6405 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6406 CI->getArgOperand(2), CI->getArgOperand(4)};
6407 NewCall = Builder.CreateCall(NewFn, Args);
6408 AttributeList OldAttrs = CI->getAttributes();
6409 AttributeList NewAttrs = AttributeList::get(
6410 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6411 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6412 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6413 NewCall->setAttributes(NewAttrs);
6414 auto *MemCI = cast<MemIntrinsic>(NewCall);
6415 // All mem intrinsics support dest alignment.
6416 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6417 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6418 // Memcpy/Memmove also support source alignment.
6419 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6420 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6421 break;
6422 }
6423
6424 case Intrinsic::masked_load:
6425 case Intrinsic::masked_gather:
6426 case Intrinsic::masked_store:
6427 case Intrinsic::masked_scatter: {
6428 if (CI->arg_size() != 4) {
6429 DefaultCase();
6430 return;
6431 }
6432
6433 auto GetMaybeAlign = [](Value *Op) {
6434 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6435 uint64_t Val = CI->getZExtValue();
6436 if (Val == 0)
6437 return MaybeAlign();
6438 if (isPowerOf2_64(Val))
6439 return MaybeAlign(Val);
6440 }
6441 reportFatalUsageError("Invalid alignment argument");
6442 };
6443 auto GetAlign = [&](Value *Op) {
6444 MaybeAlign Align = GetMaybeAlign(Op);
6445 if (Align)
6446 return *Align;
6447 reportFatalUsageError("Invalid zero alignment argument");
6448 };
6449
6450 const DataLayout &DL = CI->getDataLayout();
6451 switch (NewFn->getIntrinsicID()) {
6452 case Intrinsic::masked_load:
6453 NewCall = Builder.CreateMaskedLoad(
6454 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6455 CI->getArgOperand(2), CI->getArgOperand(3));
6456 break;
6457 case Intrinsic::masked_gather:
6458 NewCall = Builder.CreateMaskedGather(
6459 CI->getType(), CI->getArgOperand(0),
6460 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6461 CI->getType()->getScalarType()),
6462 CI->getArgOperand(2), CI->getArgOperand(3));
6463 break;
6464 case Intrinsic::masked_store:
6465 NewCall = Builder.CreateMaskedStore(
6466 CI->getArgOperand(0), CI->getArgOperand(1),
6467 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6468 break;
6469 case Intrinsic::masked_scatter:
6470 NewCall = Builder.CreateMaskedScatter(
6471 CI->getArgOperand(0), CI->getArgOperand(1),
6472 DL.getValueOrABITypeAlignment(
6473 GetMaybeAlign(CI->getArgOperand(2)),
6474 CI->getArgOperand(0)->getType()->getScalarType()),
6475 CI->getArgOperand(3));
6476 break;
6477 default:
6478 llvm_unreachable("Unexpected intrinsic ID");
6479 }
6480 // Previous metadata is still valid.
6481 NewCall->copyMetadata(*CI);
6482 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6483 break;
6484 }
6485
6486 case Intrinsic::lifetime_start:
6487 case Intrinsic::lifetime_end: {
6488 if (CI->arg_size() != 2) {
6489 DefaultCase();
6490 return;
6491 }
6492
6493 Value *Ptr = CI->getArgOperand(1);
6494 // Try to strip pointer casts, such that the lifetime works on an alloca.
6495 Ptr = Ptr->stripPointerCasts();
6496 if (isa<AllocaInst>(Ptr)) {
6497 // Don't use NewFn, as we might have looked through an addrspacecast.
6498 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6499 NewCall = Builder.CreateLifetimeStart(Ptr);
6500 else
6501 NewCall = Builder.CreateLifetimeEnd(Ptr);
6502 break;
6503 }
6504
6505 // Otherwise remove the lifetime marker.
6506 CI->eraseFromParent();
6507 return;
6508 }
6509
6510 case Intrinsic::x86_avx512_vpdpbusd_128:
6511 case Intrinsic::x86_avx512_vpdpbusd_256:
6512 case Intrinsic::x86_avx512_vpdpbusd_512:
6513 case Intrinsic::x86_avx512_vpdpbusds_128:
6514 case Intrinsic::x86_avx512_vpdpbusds_256:
6515 case Intrinsic::x86_avx512_vpdpbusds_512:
6516 case Intrinsic::x86_avx2_vpdpbssd_128:
6517 case Intrinsic::x86_avx2_vpdpbssd_256:
6518 case Intrinsic::x86_avx10_vpdpbssd_512:
6519 case Intrinsic::x86_avx2_vpdpbssds_128:
6520 case Intrinsic::x86_avx2_vpdpbssds_256:
6521 case Intrinsic::x86_avx10_vpdpbssds_512:
6522 case Intrinsic::x86_avx2_vpdpbsud_128:
6523 case Intrinsic::x86_avx2_vpdpbsud_256:
6524 case Intrinsic::x86_avx10_vpdpbsud_512:
6525 case Intrinsic::x86_avx2_vpdpbsuds_128:
6526 case Intrinsic::x86_avx2_vpdpbsuds_256:
6527 case Intrinsic::x86_avx10_vpdpbsuds_512:
6528 case Intrinsic::x86_avx2_vpdpbuud_128:
6529 case Intrinsic::x86_avx2_vpdpbuud_256:
6530 case Intrinsic::x86_avx10_vpdpbuud_512:
6531 case Intrinsic::x86_avx2_vpdpbuuds_128:
6532 case Intrinsic::x86_avx2_vpdpbuuds_256:
6533 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6534 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6535 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6536 CI->getArgOperand(2)};
6537 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6538 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6539 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6540
6541 NewCall = Builder.CreateCall(NewFn, Args);
6542 break;
6543 }
6544 case Intrinsic::x86_avx512_vpdpwssd_128:
6545 case Intrinsic::x86_avx512_vpdpwssd_256:
6546 case Intrinsic::x86_avx512_vpdpwssd_512:
6547 case Intrinsic::x86_avx512_vpdpwssds_128:
6548 case Intrinsic::x86_avx512_vpdpwssds_256:
6549 case Intrinsic::x86_avx512_vpdpwssds_512:
6550 case Intrinsic::x86_avx2_vpdpwsud_128:
6551 case Intrinsic::x86_avx2_vpdpwsud_256:
6552 case Intrinsic::x86_avx10_vpdpwsud_512:
6553 case Intrinsic::x86_avx2_vpdpwsuds_128:
6554 case Intrinsic::x86_avx2_vpdpwsuds_256:
6555 case Intrinsic::x86_avx10_vpdpwsuds_512:
6556 case Intrinsic::x86_avx2_vpdpwusd_128:
6557 case Intrinsic::x86_avx2_vpdpwusd_256:
6558 case Intrinsic::x86_avx10_vpdpwusd_512:
6559 case Intrinsic::x86_avx2_vpdpwusds_128:
6560 case Intrinsic::x86_avx2_vpdpwusds_256:
6561 case Intrinsic::x86_avx10_vpdpwusds_512:
6562 case Intrinsic::x86_avx2_vpdpwuud_128:
6563 case Intrinsic::x86_avx2_vpdpwuud_256:
6564 case Intrinsic::x86_avx10_vpdpwuud_512:
6565 case Intrinsic::x86_avx2_vpdpwuuds_128:
6566 case Intrinsic::x86_avx2_vpdpwuuds_256:
6567 case Intrinsic::x86_avx10_vpdpwuuds_512:
6568 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6569 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6570 CI->getArgOperand(2)};
6571 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6572 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6573 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6574
6575 NewCall = Builder.CreateCall(NewFn, Args);
6576 break;
6577 }
6578 assert(NewCall && "Should have either set this variable or returned through "
6579 "the default case");
6580 NewCall->takeName(CI);
6581 CI->replaceAllUsesWith(NewCall);
6582 CI->eraseFromParent();
6583}
6584
6585void llvm::UpgradeCallsToIntrinsic(Function *F) {
6586 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6587
6588 // Check if this function should be upgraded and get the replacement function
6589 // if there is one.
6590 Function *NewFn;
6591 if (UpgradeIntrinsicFunction(F, NewFn)) {
6592 // Replace all users of the old function with the new function or new
6593 // instructions. This is not a range loop because the call is deleted.
6594 for (User *U : make_early_inc_range(F->users()))
6595 if (CallBase *CB = dyn_cast<CallBase>(U))
6596 UpgradeIntrinsicCall(CB, NewFn);
6597
6598 // Remove old function, no longer used, from the module.
6599 if (F != NewFn)
6600 F->eraseFromParent();
6601 }
6602}
6603
6604MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6605 const unsigned NumOperands = MD.getNumOperands();
6606 if (NumOperands == 0)
6607 return &MD; // Invalid, punt to a verifier error.
6608
6609 // Check if the tag uses struct-path aware TBAA format.
6610 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6611 return &MD;
6612
6613 auto &Context = MD.getContext();
6614 if (NumOperands == 3) {
6615 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6616 MDNode *ScalarType = MDNode::get(Context, Elts);
6617 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6618 Metadata *Elts2[] = {ScalarType, ScalarType,
6619 ConstantAsMetadata::get(
6620 Constant::getNullValue(Type::getInt64Ty(Context))),
6621 MD.getOperand(2)};
6622 return MDNode::get(Context, Elts2);
6623 }
6624 // Create a MDNode <MD, MD, offset 0>
6625 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6626 Type::getInt64Ty(Context)))};
6627 return MDNode::get(Context, Elts);
6628}
6629
6630Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6631 Instruction *&Temp) {
6632 if (Opc != Instruction::BitCast)
6633 return nullptr;
6634
6635 Temp = nullptr;
6636 Type *SrcTy = V->getType();
6637 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6638 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6639 LLVMContext &Context = V->getContext();
6640
6641 // We have no information about target data layout, so we assume that
6642 // the maximum pointer size is 64bit.
6643 Type *MidTy = Type::getInt64Ty(Context);
6644 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6645
6646 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6647 }
6648
6649 return nullptr;
6650}
6651
6652Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6653 if (Opc != Instruction::BitCast)
6654 return nullptr;
6655
6656 Type *SrcTy = C->getType();
6657 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6658 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6659 LLVMContext &Context = C->getContext();
6660
6661 // We have no information about target data layout, so we assume that
6662 // the maximum pointer size is 64bit.
6663 Type *MidTy = Type::getInt64Ty(Context);
6664
6665 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6666 DestTy);
6667 }
6668
6669 return nullptr;
6670}
6671
6672static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6673 if (Flag.getNumOperands() < 3)
6674 return std::nullopt;
6675 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6676 return Name->getString();
6677 return std::nullopt;
6678}
6679
6680/// Check the debug info version number, if it is out-dated, drop the debug
6681/// info. Return true if module is modified.
6682bool llvm::UpgradeDebugInfo(Module &M) {
6683 if (DisableAutoUpgradeDebugInfo)
6684 return false;
6685
6686 llvm::TimeTraceScope timeScope("Upgrade debug info");
6687 // We need to get metadata before the module is verified (i.e., getModuleFlag
6688 // makes assumptions that we haven't verified yet). Carefully extract the flag
6689 // from the metadata.
6690 unsigned Version = 0;
6691 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6692 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6693 if (auto Name = getModuleFlagNameSafely(*Flag))
6694 return *Name == "Debug Info Version";
6695 return false;
6696 });
6697 if (OpIt != ModFlags->op_end()) {
6698 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6699 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6700 Version = CI->getZExtValue();
6701 }
6702 }
6703
6704 if (Version == DEBUG_METADATA_VERSION) {
6705 bool BrokenDebugInfo = false;
6706 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6707 report_fatal_error("Broken module found, compilation aborted!");
6708 if (!BrokenDebugInfo)
6709 // Everything is ok.
6710 return false;
6711 else {
6712 // Diagnose malformed debug info.
6713 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6714 M.getContext().diagnose(Diag);
6715 }
6716 }
6717 bool Modified = StripDebugInfo(M);
6718 if (Modified && Version != DEBUG_METADATA_VERSION) {
6719 // Diagnose a version mismatch.
6720 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6721 M.getContext().diagnose(DiagVersion);
6722 }
6723 return Modified;
6724}
6725
6726static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6727 GlobalValue *GV, const Metadata *V) {
6728 Function *F = cast<Function>(GV);
6729
6730 constexpr StringLiteral DefaultValue = "1";
6731 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6732 unsigned Length = 0;
6733
6734 if (F->hasFnAttribute(Attr)) {
6735 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6736 // parse these elements placing them into Vect3
6737 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6738 for (; Length < 3 && !S.empty(); Length++) {
6739 auto [Part, Rest] = S.split(',');
6740 Vect3[Length] = Part.trim();
6741 S = Rest;
6742 }
6743 }
6744
6745 const unsigned Dim = DimC - 'x';
6746 assert(Dim < 3 && "Unexpected dim char");
6747
6748 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6749
6750 // local variable required for StringRef in Vect3 to point to.
6751 const std::string VStr = llvm::utostr(VInt);
6752 Vect3[Dim] = VStr;
6753 Length = std::max(Length, Dim + 1);
6754
6755 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6756 F->addFnAttr(Attr, NewAttr);
6757}
6758
6759static inline bool isXYZ(StringRef S) {
6760 return S == "x" || S == "y" || S == "z";
6761}
6762
6763bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6764 const Metadata *V) {
6765 if (K == "kernel") {
6766 if (!mdconst::extract<ConstantInt>(V)->isZero())
6767 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6768 return true;
6769 }
6770 if (K == "align") {
6771 // V is a bitfeild specifying two 16-bit values. The alignment value is
6772 // specfied in low 16-bits, The index is specified in the high bits. For the
6773 // index, 0 indicates the return value while higher values correspond to
6774 // each parameter (idx = param + 1).
6775 const uint64_t AlignIdxValuePair =
6776 mdconst::extract<ConstantInt>(V)->getZExtValue();
6777 const unsigned Idx = (AlignIdxValuePair >> 16);
6778 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6779 cast<Function>(GV)->addAttributeAtIndex(
6780 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6781 return true;
6782 }
6783 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6784 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6785 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6786 return true;
6787 }
6788 if (K == "minctasm") {
6789 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6790 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6791 return true;
6792 }
6793 if (K == "maxnreg") {
6794 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6795 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6796 return true;
6797 }
6798 if (K.consume_front("maxntid") && isXYZ(K)) {
6799 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6800 return true;
6801 }
6802 if (K.consume_front("reqntid") && isXYZ(K)) {
6803 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6804 return true;
6805 }
6806 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6807 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6808 return true;
6809 }
6810 if (K == "grid_constant") {
6811 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6812 for (const auto &Op : cast<MDNode>(V)->operands()) {
6813 // For some reason, the index is 1-based in the metadata. Good thing we're
6814 // able to auto-upgrade it!
6815 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6816 cast<Function>(GV)->addParamAttr(Index, Attr);
6817 }
6818 return true;
6819 }
6820
6821 return false;
6822}
6823
6824void llvm::UpgradeNVVMAnnotations(Module &M) {
6825 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6826 if (!NamedMD)
6827 return;
6828
6829 SmallVector<MDNode *, 8> NewNodes;
6830 SmallPtrSet<const MDNode *, 8> SeenNodes;
6831 for (MDNode *MD : NamedMD->operands()) {
6832 if (!SeenNodes.insert(MD).second)
6833 continue;
6834
6835 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6836 if (!GV)
6837 continue;
6838
6839 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6840
6841 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6842 // Each nvvm.annotations metadata entry will be of the following form:
6843 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6844 // start index = 1, to skip the global variable key
6845 // increment = 2, to skip the value for each property-value pairs
6846 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6847 MDString *K = cast<MDString>(MD->getOperand(j));
6848 const MDOperand &V = MD->getOperand(j + 1);
6849 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6850 if (!Upgraded)
6851 NewOperands.append({K, V});
6852 }
6853
6854 if (NewOperands.size() > 1)
6855 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6856 }
6857
6858 NamedMD->clearOperands();
6859 for (MDNode *N : NewNodes)
6860 NamedMD->addOperand(N);
6861}
6862
6863/// This checks for objc retain release marker which should be upgraded. It
6864/// returns true if module is modified.
6865static bool upgradeRetainReleaseMarker(Module &M) {
6866 bool Changed = false;
6867 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6868 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6869 if (ModRetainReleaseMarker) {
6870 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6871 if (Op) {
6872 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6873 if (ID) {
6874 SmallVector<StringRef, 4> ValueComp;
6875 ID->getString().split(ValueComp, "#");
6876 if (ValueComp.size() == 2) {
6877 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6878 ID = MDString::get(M.getContext(), NewValue);
6879 }
6880 M.addModuleFlag(Module::Error, MarkerKey, ID);
6881 M.eraseNamedMetadata(ModRetainReleaseMarker);
6882 Changed = true;
6883 }
6884 }
6885 }
6886 return Changed;
6887}
6888
6889void llvm::UpgradeARCRuntime(Module &M) {
6890 // This lambda converts normal function calls to ARC runtime functions to
6891 // intrinsic calls.
6892 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6893 llvm::Intrinsic::ID IntrinsicFunc) {
6894 Function *Fn = M.getFunction(OldFunc);
6895
6896 if (!Fn)
6897 return;
6898
6899 Function *NewFn =
6900 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6901
6902 for (User *U : make_early_inc_range(Fn->users())) {
6903 CallInst *CI = dyn_cast<CallInst>(U);
6904 if (!CI || CI->getCalledFunction() != Fn)
6905 continue;
6906
6907 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6908 FunctionType *NewFuncTy = NewFn->getFunctionType();
6909 SmallVector<Value *, 2> Args;
6910
6911 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6912 // value to the return type of the old function.
6913 if (NewFuncTy->getReturnType() != CI->getType() &&
6914 !CastInst::castIsValid(Instruction::BitCast, CI,
6915 NewFuncTy->getReturnType()))
6916 continue;
6917
6918 bool InvalidCast = false;
6919
6920 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6921 Value *Arg = CI->getArgOperand(I);
6922
6923 // Bitcast argument to the parameter type of the new function if it's
6924 // not a variadic argument.
6925 if (I < NewFuncTy->getNumParams()) {
6926 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6927 // to the parameter type of the new function.
6928 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6929 NewFuncTy->getParamType(I))) {
6930 InvalidCast = true;
6931 break;
6932 }
6933 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6934 }
6935 Args.push_back(Arg);
6936 }
6937
6938 if (InvalidCast)
6939 continue;
6940
6941 // Create a call instruction that calls the new function.
6942 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6943 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6944 NewCall->takeName(CI);
6945
6946 // Bitcast the return value back to the type of the old call.
6947 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6948
6949 if (!CI->use_empty())
6950 CI->replaceAllUsesWith(NewRetVal);
6951 CI->eraseFromParent();
6952 }
6953
6954 if (Fn->use_empty())
6955 Fn->eraseFromParent();
6956 };
6957
6958 // Unconditionally convert a call to "clang.arc.use" to a call to
6959 // "llvm.objc.clang.arc.use".
6960 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6961
6962 // Upgrade the retain release marker. If there is no need to upgrade
6963 // the marker, that means either the module is already new enough to contain
6964 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6965 if (!upgradeRetainReleaseMarker(M))
6966 return;
6967
6968 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6969 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6970 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6971 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6972 {"objc_autoreleaseReturnValue",
6973 llvm::Intrinsic::objc_autoreleaseReturnValue},
6974 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6975 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6976 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6977 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6978 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6979 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6980 {"objc_release", llvm::Intrinsic::objc_release},
6981 {"objc_retain", llvm::Intrinsic::objc_retain},
6982 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6983 {"objc_retainAutoreleaseReturnValue",
6984 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6985 {"objc_retainAutoreleasedReturnValue",
6986 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6987 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6988 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6989 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6990 {"objc_unsafeClaimAutoreleasedReturnValue",
6991 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6992 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6993 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6994 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6995 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6996 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6997 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6998 {"objc_arc_annotation_topdown_bbstart",
6999 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7000 {"objc_arc_annotation_topdown_bbend",
7001 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7002 {"objc_arc_annotation_bottomup_bbstart",
7003 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7004 {"objc_arc_annotation_bottomup_bbend",
7005 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7006
7007 for (auto &I : RuntimeFuncs)
7008 UpgradeToIntrinsic(I.first, I.second);
7009}
7010
7011// Upgrade the way signing of pointers to init/fini functions is described.
7012//
7013// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7014// constants, if signing was requested. After the upgrade, these arrays contain
7015// plain function pointers and the desired signing schema is described via a
7016// pair of module flags.
7017//
7018// Note that the upgrade is only performed if all elements of *both* arrays
7019// agree on a common signing schema.
7020static bool upgradePtrauthInitFiniArrays(Module &M) {
7021 // As we cannot always decide whether the particular module should have
7022 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7023 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7024 // returns as soon as it spots any non-signed init/fini pointer: either we
7025 // should request non-signed pointers (safe to omit both flags) or there is
7026 // no common schema (and thus we do not modify anything).
7027 //
7028 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7029 // or whether we should request address diversity in addition to the basic
7030 // constant diversity. There is no value representing "decided not to sign"
7031 // for the reasons explained above.
7032 std::optional<bool> UseAddressDisc;
7033
7034 // Do not attempt upgrading if the new module flags already exist.
7035 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7036 for (const MDNode *Flag : ModFlags->operands()) {
7037 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7038 if (Name && (*Name == "ptrauth-init-fini" ||
7039 *Name == "ptrauth-init-fini-address-discrimination"))
7040 return false;
7041 }
7042 }
7043
7044 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7045 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7046 constexpr unsigned ExpectedAddressMarker = 1;
7047
7048 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7049 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7050 return nullptr; // Nothing to upgrade or unknown pattern found.
7051
7052 bool HasAddressDisc;
7053 if (!CPA->hasAddressDiscriminator())
7054 HasAddressDisc = false;
7055 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7056 HasAddressDisc = true;
7057 else
7058 return nullptr; // Unknown pattern.
7059
7060 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7061 return nullptr; // Disagreement with the decided mode.
7062
7063 UseAddressDisc = HasAddressDisc;
7064 return CPA->getPointer();
7065 };
7066
7067 // Do not apply any changes until we know the upgrade is non-ambiguous.
7068 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7069 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7070
7071 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7072 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7073 if (!GV || !GV->hasInitializer())
7074 continue; // Skip, but it is okay to upgrade the other variable.
7075
7076 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7077 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7078 return false;
7079
7080 std::vector<Constant *> NewStructors;
7081 NewStructors.reserve(OldStructorsArray->getNumOperands());
7082
7083 for (Use &U : OldStructorsArray->operands()) {
7084 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7085 if (!Structor || Structor->getNumOperands() != 3)
7086 return false;
7087
7088 Constant *Prio = Structor->getOperand(0);
7089 Constant *Func = Structor->getOperand(1);
7090 Constant *Arg = Structor->getOperand(2);
7091
7092 Func = UpgradeSinglePointer(Func);
7093 if (!Func)
7094 return false;
7095
7096 NewStructors.push_back(
7097 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7098 }
7099
7100 Constant *NewInit =
7101 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7102 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7103 }
7104
7105 if (GlobalArraysToUpgrade.empty())
7106 return false;
7107 assert(UseAddressDisc.has_value());
7108
7109 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7110 GV->setInitializer(NewInit);
7111
7112 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7113 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7114 *UseAddressDisc);
7115
7116 return true;
7117}
7118
7119bool llvm::UpgradeModuleFlags(Module &M) {
7120 bool Changed = false;
7121 Changed |= upgradePtrauthInitFiniArrays(M);
7122
7123 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7124 if (!ModFlags)
7125 return Changed;
7126
7127 bool HasObjCFlag = false, HasClassProperties = false;
7128 bool HasSwiftVersionFlag = false;
7129 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7130 uint32_t SwiftABIVersion;
7131 auto Int8Ty = Type::getInt8Ty(M.getContext());
7132 auto Int32Ty = Type::getInt32Ty(M.getContext());
7133
7134 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7135 MDNode *Op = ModFlags->getOperand(I);
7136 if (Op->getNumOperands() != 3)
7137 continue;
7138 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7139 if (!ID)
7140 continue;
7141 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7142 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7143 Type::getInt32Ty(M.getContext()), B)),
7144 MDString::get(M.getContext(), ID->getString()),
7145 Op->getOperand(2)};
7146 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7147 Changed = true;
7148 };
7149
7150 if (ID->getString() == "Objective-C Image Info Version")
7151 HasObjCFlag = true;
7152 if (ID->getString() == "Objective-C Class Properties")
7153 HasClassProperties = true;
7154 // Upgrade PIC from Error/Max to Min.
7155 if (ID->getString() == "PIC Level") {
7156 if (auto *Behavior =
7157 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7158 uint64_t V = Behavior->getLimitedValue();
7159 if (V == Module::Error || V == Module::Max)
7160 SetBehavior(Module::Min);
7161 }
7162 }
7163 // Upgrade "PIE Level" from Error to Max.
7164 if (ID->getString() == "PIE Level")
7165 if (auto *Behavior =
7166 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7167 if (Behavior->getLimitedValue() == Module::Error)
7168 SetBehavior(Module::Max);
7169
7170 // Upgrade branch protection and return address signing module flags. The
7171 // module flag behavior for these fields were Error and now they are Min.
7172 if (ID->getString() == "branch-target-enforcement" ||
7173 ID->getString().starts_with("sign-return-address")) {
7174 if (auto *Behavior =
7175 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7176 if (Behavior->getLimitedValue() == Module::Error) {
7177 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7178 Metadata *Ops[3] = {
7179 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7180 Op->getOperand(1), Op->getOperand(2)};
7181 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7182 Changed = true;
7183 }
7184 }
7185 }
7186
7187 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7188 // section name so that llvm-lto will not complain about mismatching
7189 // module flags that is functionally the same.
7190 if (ID->getString() == "Objective-C Image Info Section") {
7191 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7192 SmallVector<StringRef, 4> ValueComp;
7193 Value->getString().split(ValueComp, " ");
7194 if (ValueComp.size() != 1) {
7195 std::string NewValue;
7196 for (auto &S : ValueComp)
7197 NewValue += S.str();
7198 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7199 MDString::get(M.getContext(), NewValue)};
7200 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7201 Changed = true;
7202 }
7203 }
7204 }
7205
7206 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7207 // If the higher bits are set, it adds new module flag for swift info.
7208 if (ID->getString() == "Objective-C Garbage Collection") {
7209 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7210 if (Md) {
7211 assert(Md->getValue() && "Expected non-empty metadata");
7212 auto Type = Md->getValue()->getType();
7213 if (Type == Int8Ty)
7214 continue;
7215 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7216 if ((Val & 0xff) != Val) {
7217 HasSwiftVersionFlag = true;
7218 SwiftABIVersion = (Val & 0xff00) >> 8;
7219 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7220 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7221 }
7222 Metadata *Ops[3] = {
7223 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7224 Op->getOperand(1),
7225 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7226 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7227 Changed = true;
7228 }
7229 }
7230
7231 if (ID->getString() == "amdgpu_code_object_version") {
7232 Metadata *Ops[3] = {
7233 Op->getOperand(0),
7234 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7235 Op->getOperand(2)};
7236 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7237 Changed = true;
7238 }
7239
7240 // clang/PowerPC used to use "float-abi" to describe the long double format;
7241 // it has been renamed to "long-double-type", with its values changed to the
7242 // corresponding IR floating-point type names.
7243 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7244 StringRef Format;
7245 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7246 Format = S->getString();
7247
7248 // The "float-abi" key is now reserved for the target-independent
7249 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7250 // (including unrecognized ones, which were never valid) to the default.
7251 if (!FloatABI::parseABIType(Format)) {
7252 LongDoubleFormat NewFormat =
7253 StringSwitch<LongDoubleFormat>(Format)
7254 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7255 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7256 .Default(LongDoubleFormat::PPCDoubleDouble);
7257 Metadata *Ops[3] = {
7258 Op->getOperand(0),
7259 MDString::get(M.getContext(), "long-double-type"),
7260 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7261 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7262 Changed = true;
7263 }
7264 }
7265 }
7266
7267 // "Objective-C Class Properties" is recently added for Objective-C. We
7268 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7269 // flag of value 0, so we can correclty downgrade this flag when trying to
7270 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7271 // this module flag.
7272 if (HasObjCFlag && !HasClassProperties) {
7273 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7274 (uint32_t)0);
7275 Changed = true;
7276 }
7277
7278 if (HasSwiftVersionFlag) {
7279 M.addModuleFlag(Module::Error, "Swift ABI Version",
7280 SwiftABIVersion);
7281 M.addModuleFlag(Module::Error, "Swift Major Version",
7282 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7283 M.addModuleFlag(Module::Error, "Swift Minor Version",
7284 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7285 Changed = true;
7286 }
7287
7288 return Changed;
7289}
7290
7291bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7292 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7293 // If this metadata has operands, we expect all of them to be either from
7294 // before or from after the format change handled here, so we can bail out
7295 // fast if the first (if any) operands is of the new format.
7296 auto MatchesVersion = [](const MDNode *Op) {
7297 return Op->getNumOperands() >= 3 &&
7298 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7299 cast<ConstantAsMetadata>(Op->getOperand(2))
7300 ->getType()
7301 ->isIntegerTy(64);
7302 };
7303
7304 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7305 MatchesVersion(CFIConsts->getOperand(0)))
7306 return false;
7307
7308 bool Changed = false;
7309 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7310 MDNode *Op = CFIConsts->getOperand(I);
7311 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7312 assert(Op->getNumOperands() >= 2 &&
7313 "Expected at least 2 operands - name and linkage type");
7314 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7315 StringRef Name = NameMD->getString();
7316 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7317 GlobalValue::dropLLVMManglingEscape(Name));
7318
7319 SmallVector<Metadata *, 4> Elts;
7320 Elts.push_back(Op->getOperand(0));
7321 Elts.push_back(Op->getOperand(1));
7322 Elts.push_back(ConstantAsMetadata::get(
7323 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7324
7325 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7326 Elts.push_back(Op->getOperand(J));
7327
7328 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7329 Changed = true;
7330 }
7331
7332 return Changed;
7333}
7334
7335void llvm::UpgradeSectionAttributes(Module &M) {
7336 auto TrimSpaces = [](StringRef Section) -> std::string {
7337 SmallVector<StringRef, 5> Components;
7338 Section.split(Components, ',');
7339
7340 SmallString<32> Buffer;
7341 raw_svector_ostream OS(Buffer);
7342
7343 for (auto Component : Components)
7344 OS << ',' << Component.trim();
7345
7346 return std::string(OS.str().substr(1));
7347 };
7348
7349 for (auto &GV : M.globals()) {
7350 if (!GV.hasSection())
7351 continue;
7352
7353 StringRef Section = GV.getSection();
7354
7355 if (!Section.starts_with("__DATA, __objc_catlist"))
7356 continue;
7357
7358 // __DATA, __objc_catlist, regular, no_dead_strip
7359 // __DATA,__objc_catlist,regular,no_dead_strip
7360 GV.setSection(TrimSpaces(Section));
7361 }
7362}
7363
7364namespace {
7365// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7366// callsites within a function that did not also have the strictfp attribute.
7367// Since 10.0, if strict FP semantics are needed within a function, the
7368// function must have the strictfp attribute and all calls within the function
7369// must also have the strictfp attribute. This latter restriction is
7370// necessary to prevent unwanted libcall simplification when a function is
7371// being cloned (such as for inlining).
7372//
7373// The "dangling" strictfp attribute usage was only used to prevent constant
7374// folding and other libcall simplification. The nobuiltin attribute on the
7375// callsite has the same effect.
7376struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7377 StrictFPUpgradeVisitor() = default;
7378
7379 void visitCallBase(CallBase &Call) {
7380 if (!Call.isStrictFP())
7381 return;
7382 if (isa<ConstrainedFPIntrinsic>(&Call))
7383 return;
7384 // If we get here, the caller doesn't have the strictfp attribute
7385 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7386 Call.removeFnAttr(Attribute::StrictFP);
7387 Call.addFnAttr(Attribute::NoBuiltin);
7388 }
7389};
7390
7391/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7392struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7393 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7394 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7395
7396 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7397 if (!RMW.isFloatingPointOperation())
7398 return;
7399
7400 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7401 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7402 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7403 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
7404 }
7405};
7406} // namespace
7407
7408void llvm::UpgradeFunctionAttributes(Function &F) {
7409 // If a function definition doesn't have the strictfp attribute,
7410 // convert any callsite strictfp attributes to nobuiltin.
7411 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7412 StrictFPUpgradeVisitor SFPV;
7413 SFPV.visit(F);
7414 }
7415
7416 // Remove all incompatibile attributes from function.
7417 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7418 F.getReturnType(), F.getAttributes().getRetAttrs()));
7419 for (auto &Arg : F.args())
7420 Arg.removeAttrs(
7421 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7422
7423 bool AddingAttrs = false, RemovingAttrs = false;
7424 AttrBuilder AttrsToAdd(F.getContext());
7425 AttributeMask AttrsToRemove;
7426
7427 // Older versions of LLVM treated an "implicit-section-name" attribute
7428 // similarly to directly setting the section on a Function.
7429 if (Attribute A = F.getFnAttribute("implicit-section-name");
7430 A.isValid() && A.isStringAttribute()) {
7431 F.setSection(A.getValueAsString());
7432 AttrsToRemove.addAttribute("implicit-section-name");
7433 RemovingAttrs = true;
7434 }
7435
7436 if (Attribute A = F.getFnAttribute("nooutline");
7437 A.isValid() && A.isStringAttribute()) {
7438 AttrsToRemove.addAttribute("nooutline");
7439 AttrsToAdd.addAttribute(Attribute::NoOutline);
7440 AddingAttrs = RemovingAttrs = true;
7441 }
7442
7443 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7444 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7445 AttrsToRemove.addAttribute("uniform-work-group-size");
7446 RemovingAttrs = true;
7447 if (A.getValueAsString() == "true") {
7448 AttrsToAdd.addAttribute("uniform-work-group-size");
7449 AddingAttrs = true;
7450 }
7451 }
7452
7453 if (!F.empty()) {
7454 // For some reason this is called twice, and the first time is before any
7455 // instructions are loaded into the body.
7456
7457 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7458 A.isValid()) {
7459
7460 if (A.getValueAsBool()) {
7461 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7462 Visitor.visit(F);
7463 }
7464
7465 // We will leave behind dead attribute uses on external declarations, but
7466 // clang never added these to declarations anyway.
7467 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7468 RemovingAttrs = true;
7469 }
7470 }
7471
7472 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7473 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7474
7475 bool HandleDenormalMode = false;
7476
7477 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7478 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7479 if (ParsedMode.isValid()) {
7480 DenormalFPMath = ParsedMode;
7481 AttrsToRemove.addAttribute("denormal-fp-math");
7482 AddingAttrs = RemovingAttrs = true;
7483 HandleDenormalMode = true;
7484 }
7485 }
7486
7487 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7488 Attr.isValid()) {
7489 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7490 if (ParsedMode.isValid()) {
7491 DenormalFPMathF32 = ParsedMode;
7492 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7493 AddingAttrs = RemovingAttrs = true;
7494 HandleDenormalMode = true;
7495 }
7496 }
7497
7498 if (HandleDenormalMode)
7499 AttrsToAdd.addDenormalFPEnvAttr(
7500 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7501
7502 if (RemovingAttrs)
7503 F.removeFnAttrs(AttrsToRemove);
7504
7505 if (AddingAttrs)
7506 F.addFnAttrs(AttrsToAdd);
7507}
7508
7509// Check if the function attribute is not present and set it.
7510static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7511 StringRef Value) {
7512 if (!F.hasFnAttribute(FnAttrName))
7513 F.addFnAttr(FnAttrName, Value);
7514}
7515
7516// Check if the function attribute is not present and set it if needed.
7517// If the attribute is "false" then removes it.
7518// If the attribute is "true" resets it to a valueless attribute.
7519static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7520 if (!F.hasFnAttribute(FnAttrName)) {
7521 if (Set)
7522 F.addFnAttr(FnAttrName);
7523 } else {
7524 auto A = F.getFnAttribute(FnAttrName);
7525 if ("false" == A.getValueAsString())
7526 F.removeFnAttr(FnAttrName);
7527 else if ("true" == A.getValueAsString()) {
7528 F.removeFnAttr(FnAttrName);
7529 F.addFnAttr(FnAttrName);
7530 }
7531 }
7532}
7533
7534void llvm::copyModuleAttrToFunctions(Module &M) {
7535 Triple T(M.getTargetTriple());
7536 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7537 return;
7538
7539 uint64_t BTEValue = 0;
7540 uint64_t BPPLRValue = 0;
7541 uint64_t GCSValue = 0;
7542 uint64_t SRAValue = 0;
7543 uint64_t SRAALLValue = 0;
7544 uint64_t SRABKeyValue = 0;
7545
7546 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7547 if (ModFlags) {
7548 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7549 MDNode *Op = ModFlags->getOperand(I);
7550 if (Op->getNumOperands() != 3)
7551 continue;
7552
7553 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7554 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7555 if (!ID || !CI)
7556 continue;
7557
7558 StringRef IDStr = ID->getString();
7559 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7560 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7561 : IDStr == "guarded-control-stack" ? &GCSValue
7562 : IDStr == "sign-return-address" ? &SRAValue
7563 : IDStr == "sign-return-address-all" ? &SRAALLValue
7564 : IDStr == "sign-return-address-with-bkey"
7565 ? &SRABKeyValue
7566 : nullptr;
7567 if (!ValPtr)
7568 continue;
7569
7570 *ValPtr = CI->getZExtValue();
7571 if (*ValPtr == 2)
7572 return;
7573 }
7574 }
7575
7576 bool BTE = BTEValue == 1;
7577 bool BPPLR = BPPLRValue == 1;
7578 bool GCS = GCSValue == 1;
7579 bool SRA = SRAValue == 1;
7580
7581 StringRef SignTypeValue = "non-leaf";
7582 if (SRA && SRAALLValue == 1)
7583 SignTypeValue = "all";
7584
7585 StringRef SignKeyValue = "a_key";
7586 if (SRA && SRABKeyValue == 1)
7587 SignKeyValue = "b_key";
7588
7589 for (Function &F : M.getFunctionList()) {
7590 if (F.isDeclaration())
7591 continue;
7592
7593 if (SRA) {
7594 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7595 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7596 } else {
7597 if (auto A = F.getFnAttribute("sign-return-address");
7598 A.isValid() && "none" == A.getValueAsString()) {
7599 F.removeFnAttr("sign-return-address");
7600 F.removeFnAttr("sign-return-address-key");
7601 }
7602 }
7603 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7604 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7605 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7606 }
7607
7608 if (BTE)
7609 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7610 if (BPPLR)
7611 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7612 if (GCS)
7613 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7614 if (SRA) {
7615 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7616 if (SRAALLValue == 1)
7617 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7618 if (SRABKeyValue == 1)
7619 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7620 }
7621}
7622
7623/// Return the replacement tags if \p T still uses a removed two-operand form.
7624static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7625 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7626 return nullptr;
7627 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7628 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7629}
7630
7631/// Build the single-operand node that replaces a boolean operand: nonzero
7632/// selects the enable tag, zero the disable tag.
7633static Metadata *makeBooleanLoopNode(LLVMContext &C,
7634 const BooleanLoopTags &Tags,
7635 const MDOperand &Op) {
7636 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7637 return MDTuple::get(C,
7638 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7639}
7640
7641static bool isOldLoopArgument(Metadata *MD) {
7642 auto *T = dyn_cast_or_null<MDTuple>(MD);
7643 if (!T)
7644 return false;
7645 if (T->getNumOperands() < 1)
7646 return false;
7647 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7648 if (!S)
7649 return false;
7650 if (S->getString().starts_with("llvm.vectorizer."))
7651 return true;
7652 return getOldBooleanLoopTags(T) != nullptr;
7653}
7654
7655static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7656 StringRef OldPrefix = "llvm.vectorizer.";
7657 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7658
7659 if (OldTag == "llvm.vectorizer.unroll")
7660 return MDString::get(C, "llvm.loop.interleave.count");
7661
7662 return MDString::get(
7663 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7664 .str());
7665}
7666
7667static Metadata *upgradeLoopArgument(Metadata *MD) {
7668 auto *T = dyn_cast_or_null<MDTuple>(MD);
7669 if (!T)
7670 return MD;
7671 if (T->getNumOperands() < 1)
7672 return MD;
7673 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7674 if (!OldTag)
7675 return MD;
7676
7677 LLVMContext &C = T->getContext();
7678
7679 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7680 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7681 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7682
7683 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7684 return MD;
7685
7686 // This has an old tag. Upgrade it.
7687 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7688
7689 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7690 // vectorize.enable/disable pair, not a two-operand enable node.
7691 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7692 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7693 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7694
7695 SmallVector<Metadata *, 8> Ops;
7696 Ops.reserve(T->getNumOperands());
7697 Ops.push_back(NewTag);
7698 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7699 Ops.push_back(T->getOperand(I));
7700
7701 return MDTuple::get(C, Ops);
7702}
7703
7704MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7705 auto *T = dyn_cast<MDTuple>(&N);
7706 if (!T)
7707 return &N;
7708
7709 if (none_of(T->operands(), isOldLoopArgument))
7710 return &N;
7711
7712 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7713 // any MDNode carrying those tags with more than one operand, so a leftover
7714 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7715 // In-place mutation is safe on distinct MDNodes.
7716 if (T->isDistinct()) {
7717 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7718 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7719 if (OpT && getOldBooleanLoopTags(OpT))
7720 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7721 }
7722 if (none_of(T->operands(), isOldLoopArgument))
7723 return &N;
7724 }
7725
7726 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7727 // attachment; the original distinct loop-ID is kept as the first operand.
7728 SmallVector<Metadata *, 8> Ops;
7729 Ops.reserve(T->getNumOperands());
7730 for (Metadata *MD : T->operands())
7731 Ops.push_back(upgradeLoopArgument(MD));
7732
7733 return MDTuple::get(T->getContext(), Ops);
7734}
7735
7736std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7737 Triple T(TT);
7738 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7739 // the address space of globals to 1. This does not apply to SPIRV Logical.
7740 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7741 !DL.contains("-G") && !DL.starts_with("G")) {
7742 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7743 }
7744
7745 if (T.isLoongArch64() || T.isRISCV64()) {
7746 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7747 auto I = DL.find("-n64-");
7748 if (I != StringRef::npos)
7749 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7750 return DL.str();
7751 }
7752
7753 // AMDGPU data layout upgrades.
7754 std::string Res = DL.str();
7755 if (T.isAMDGPU()) {
7756 // Define address spaces for constants.
7757 if (!DL.contains("-G") && !DL.starts_with("G"))
7758 Res.append(Res.empty() ? "G1" : "-G1");
7759
7760 // AMDGCN data layout upgrades.
7761 if (T.isAMDGCN()) {
7762
7763 // Add missing non-integral declarations.
7764 // This goes before adding new address spaces to prevent incoherent string
7765 // values.
7766 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7767 Res.append("-ni:7:8:9");
7768 // Update ni:7 to ni:7:8:9.
7769 if (DL.ends_with("ni:7"))
7770 Res.append(":8:9");
7771 if (DL.ends_with("ni:7:8"))
7772 Res.append(":9");
7773
7774 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7775 // resources) An empty data layout has already been upgraded to G1 by now.
7776 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7777 Res.append("-p7:160:256:256:32");
7778 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7779 Res.append("-p8:128:128:128:48");
7780 constexpr StringRef OldP8("-p8:128:128-");
7781 if (DL.contains(OldP8))
7782 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7783 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7784 Res.append("-p9:192:256:256:32");
7785
7786 // Add sizing for address space 10 through 15.
7787 // AS 10-14 are reserved and defaulted to 32:32
7788 // AS 15 is in use and is 32:32.
7789 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7790 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7791 Res.append(("-" + AS + ":32:32").str());
7792 }
7793 }
7794
7795 // Upgrade the ELF mangling mode.
7796 if (!DL.contains("m:e"))
7797 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7798
7799 return Res;
7800 }
7801
7802 if (T.isSystemZ() && !DL.empty()) {
7803 // Make sure the stack alignment is present.
7804 if (!DL.contains("-S64"))
7805 return "E-S64" + DL.drop_front(1).str();
7806 return DL.str();
7807 }
7808
7809 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7810 // If the datalayout matches the expected format, add pointer size address
7811 // spaces to the datalayout.
7812 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7813 if (!DL.contains(AddrSpaces)) {
7814 SmallVector<StringRef, 4> Groups;
7815 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7816 if (R.match(Res, &Groups))
7817 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7818 }
7819 };
7820
7821 // AArch64 data layout upgrades.
7822 if (T.isAArch64()) {
7823 // Add "-Fn32"
7824 if (!DL.empty() && !DL.contains("-Fn32"))
7825 Res.append("-Fn32");
7826 AddPtr32Ptr64AddrSpaces();
7827 return Res;
7828 }
7829
7830 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7831 T.isWasm()) {
7832 // Mips64 with o32 ABI did not add "-i128:128".
7833 // Add "-i128:128"
7834 std::string I64 = "-i64:64";
7835 std::string I128 = "-i128:128";
7836 if (!StringRef(Res).contains(I128)) {
7837 size_t Pos = Res.find(I64);
7838 if (Pos != size_t(-1))
7839 Res.insert(Pos + I64.size(), I128);
7840 }
7841 }
7842
7843 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7844 size_t Pos = Res.find("-S128");
7845 if (Pos == StringRef::npos)
7846 Pos = Res.size();
7847 Res.insert(Pos, "-f64:32:64");
7848 }
7849
7850 if (!T.isX86())
7851 return Res;
7852
7853 AddPtr32Ptr64AddrSpaces();
7854
7855 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7856 // for i128 operations prior to this being reflected in the data layout, and
7857 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7858 // boundaries, so although this is a breaking change, the upgrade is expected
7859 // to fix more IR than it breaks.
7860 // Intel MCU is an exception and uses 4-byte-alignment.
7861 if (!T.isOSIAMCU()) {
7862 std::string I128 = "-i128:128";
7863 if (StringRef Ref = Res; !Ref.contains(I128)) {
7864 SmallVector<StringRef, 4> Groups;
7865 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7866 if (R.match(Res, &Groups))
7867 Res = (Groups[1] + I128 + Groups[3]).str();
7868 }
7869 }
7870
7871 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7872 // Raising the alignment is safe because Clang did not produce f80 values in
7873 // the MSVC environment before this upgrade was added.
7874 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7875 StringRef Ref = Res;
7876 auto I = Ref.find("-f80:32-");
7877 if (I != StringRef::npos)
7878 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7879 }
7880
7881 return Res;
7882}
7883
7884void llvm::UpgradeAttributes(AttrBuilder &B) {
7885 StringRef FramePointer;
7886 Attribute A = B.getAttribute("no-frame-pointer-elim");
7887 if (A.isValid()) {
7888 // The value can be "true" or "false".
7889 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7890 B.removeAttribute("no-frame-pointer-elim");
7891 }
7892 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7893 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7894 if (FramePointer != "all")
7895 FramePointer = "non-leaf";
7896 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7897 }
7898 if (!FramePointer.empty())
7899 B.addAttribute("frame-pointer", FramePointer);
7900
7901 A = B.getAttribute("null-pointer-is-valid");
7902 if (A.isValid()) {
7903 // The value can be "true" or "false".
7904 bool NullPointerIsValid = A.getValueAsString() == "true";
7905 B.removeAttribute("null-pointer-is-valid");
7906 if (NullPointerIsValid)
7907 B.addAttribute(Attribute::NullPointerIsValid);
7908 }
7909
7910 A = B.getAttribute("uniform-work-group-size");
7911 if (A.isValid()) {
7912 StringRef Val = A.getValueAsString();
7913 if (!Val.empty()) {
7914 bool IsTrue = Val == "true";
7915 B.removeAttribute("uniform-work-group-size");
7916 if (IsTrue)
7917 B.addAttribute("uniform-work-group-size");
7918 }
7919 }
7920}
7921
7922void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7923 // clang.arc.attachedcall bundles are now required to have an operand.
7924 // If they don't, it's okay to drop them entirely: when there is an operand,
7925 // the "attachedcall" is meaningful and required, but without an operand,
7926 // it's just a marker NOP. Dropping it merely prevents an optimization.
7927 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7928 return OBD.getTag() == "clang.arc.attachedcall" &&
7929 OBD.inputs().empty();
7930 });
7931}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:524
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:525
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2598
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1440
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7667
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1501
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6759
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1650
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2438
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1303
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5585
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1270
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1283
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6865
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2699
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2925
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1598
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2673
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1551
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7510
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1374
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6763
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5445
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7655
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5545
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6726
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2553
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2810
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1594
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2524
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5082
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2911
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7624
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2850
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1354
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6672
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1508
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7633
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2472
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1514
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1199
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1329
getNVVMFAddUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFAddUpgrade(StringRef Name)
Definition AutoUpgrade.cpp:1474
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7641
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5152
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5515
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5242
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2789
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5457
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5616
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2504
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2536
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1636
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2738
shouldUpgradeNVPTXTcgen05MMAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1461
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1174
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2872
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2658
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3182
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2904
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2934
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5467
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7519
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2820
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1618
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2771
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1242
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5528
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7020
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3390
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5563
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:776
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:936
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:788
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:792
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:813
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:852
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:806
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:796
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:802
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:856
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:844
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:824
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:820
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:848
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:983
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:511
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:359
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1597
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1717
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2463
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2395
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2449
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3474
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2908
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:356
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:554
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:352
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1805
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:703
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:351
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1525
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:629
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:611
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:107
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1589
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1581
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1577
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1596
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1587
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2097
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:592
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:993
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:718
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1252
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5333
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1423
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:801
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:649
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:636
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1404
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1397
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5677
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7335
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3173
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2386
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7704
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7884
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6585
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6824
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7119
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7291
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7534
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7922
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6652
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7736
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2405
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:322
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6630
llvm::RecurKind::FAdd
@ FAdd
Sum of floats.
Definition IVDescriptors.h:49
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1947
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6682
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7408
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6604
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6889
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7848
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0