LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/Value.h"
47#include "llvm/IR/Verifier.h"
48#include "llvm/Support/AMDGPUAddrSpace.h"
49#include "llvm/Support/CodeGen.h"
50#include "llvm/Support/CommandLine.h"
51#include "llvm/Support/ErrorHandling.h"
52#include "llvm/Support/NVPTXAddrSpace.h"
53#include "llvm/Support/NVVMAttributes.h"
54#include "llvm/Support/Regex.h"
55#include "llvm/Support/TimeProfiler.h"
56#include "llvm/TargetParser/Triple.h"
57#include <cstdint>
58#include <cstring>
59#include <numeric>
60
61using namespace llvm;
62
63static cl::opt<bool>
64 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
65 cl::desc("Disable autoupgrade of debug info"));
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(0);
85 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(F);
90 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(F);
106 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(F);
119 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
134 return false;
135
136 rename(F);
137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
152 return false;
153
154 rename(F);
155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(F);
165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(F);
175 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front("avx."))
186 return (Name.starts_with("blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with("movnt.") || // Added in 3.2
191 Name.starts_with("sqrt.p") || // Added in 7.0
192 Name.starts_with("storeu.") || // Added in 3.9
193 Name.starts_with("vbroadcast.s") || // Added in 3.5
194 Name.starts_with("vbroadcastf128") || // Added in 4.0
195 Name.starts_with("vextractf128.") || // Added in 3.7
196 Name.starts_with("vinsertf128.") || // Added in 3.7
197 Name.starts_with("vperm2f128.") || // Added in 6.0
198 Name.starts_with("vpermil.")); // Added in 3.1
199
200 if (Name.consume_front("avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with("pabs.") || // Added in 6.0
203 Name.starts_with("padds.") || // Added in 8.0
204 Name.starts_with("paddus.") || // Added in 8.0
205 Name.starts_with("pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with("pbroadcast") || // Added in 3.8
208 Name.starts_with("pcmpeq.") || // Added in 3.1
209 Name.starts_with("pcmpgt.") || // Added in 3.1
210 Name.starts_with("pmax") || // Added in 3.9
211 Name.starts_with("pmin") || // Added in 3.9
212 Name.starts_with("pmovsx") || // Added in 3.9
213 Name.starts_with("pmovzx") || // Added in 3.9
214 Name == "pmul.dq" || // Added in 7.0
215 Name == "pmulu.dq" || // Added in 7.0
216 Name.starts_with("psll.dq") || // Added in 3.7
217 Name.starts_with("psrl.dq") || // Added in 3.7
218 Name.starts_with("psubs.") || // Added in 8.0
219 Name.starts_with("psubus.") || // Added in 8.0
220 Name.starts_with("vbroadcast") || // Added in 3.8
221 Name == "vbroadcasti128" || // Added in 3.7
222 Name == "vextracti128" || // Added in 3.7
223 Name == "vinserti128" || // Added in 3.7
224 Name == "vperm2i128"); // Added in 6.0
225
226 if (Name.consume_front("avx512.")) {
227 if (Name.consume_front("mask."))
228 // 'avx512.mask.*'
229 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
230 Name.starts_with("and.") || // Added in 3.9
231 Name.starts_with("andn.") || // Added in 3.9
232 Name.starts_with("broadcast.s") || // Added in 3.9
233 Name.starts_with("broadcastf32x4.") || // Added in 6.0
234 Name.starts_with("broadcastf32x8.") || // Added in 6.0
235 Name.starts_with("broadcastf64x2.") || // Added in 6.0
236 Name.starts_with("broadcastf64x4.") || // Added in 6.0
237 Name.starts_with("broadcasti32x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x8.") || // Added in 6.0
239 Name.starts_with("broadcasti64x2.") || // Added in 6.0
240 Name.starts_with("broadcasti64x4.") || // Added in 6.0
241 Name.starts_with("cmp.b") || // Added in 5.0
242 Name.starts_with("cmp.d") || // Added in 5.0
243 Name.starts_with("cmp.q") || // Added in 5.0
244 Name.starts_with("cmp.w") || // Added in 5.0
245 Name.starts_with("compress.b") || // Added in 9.0
246 Name.starts_with("compress.d") || // Added in 9.0
247 Name.starts_with("compress.p") || // Added in 9.0
248 Name.starts_with("compress.q") || // Added in 9.0
249 Name.starts_with("compress.store.") || // Added in 7.0
250 Name.starts_with("compress.w") || // Added in 9.0
251 Name.starts_with("conflict.") || // Added in 9.0
252 Name.starts_with("cvtdq2pd.") || // Added in 4.0
253 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
254 Name == "cvtpd2dq.256" || // Added in 7.0
255 Name == "cvtpd2ps.256" || // Added in 7.0
256 Name == "cvtps2pd.128" || // Added in 7.0
257 Name == "cvtps2pd.256" || // Added in 7.0
258 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
259 Name == "cvtqq2ps.256" || // Added in 9.0
260 Name == "cvtqq2ps.512" || // Added in 9.0
261 Name == "cvttpd2dq.256" || // Added in 7.0
262 Name == "cvttps2dq.128" || // Added in 7.0
263 Name == "cvttps2dq.256" || // Added in 7.0
264 Name.starts_with("cvtudq2pd.") || // Added in 4.0
265 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
266 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
267 Name == "cvtuqq2ps.256" || // Added in 9.0
268 Name == "cvtuqq2ps.512" || // Added in 9.0
269 Name.starts_with("dbpsadbw.") || // Added in 7.0
270 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
271 Name.starts_with("expand.b") || // Added in 9.0
272 Name.starts_with("expand.d") || // Added in 9.0
273 Name.starts_with("expand.load.") || // Added in 7.0
274 Name.starts_with("expand.p") || // Added in 9.0
275 Name.starts_with("expand.q") || // Added in 9.0
276 Name.starts_with("expand.w") || // Added in 9.0
277 Name.starts_with("fpclass.p") || // Added in 7.0
278 Name.starts_with("insert") || // Added in 4.0
279 Name.starts_with("load.") || // Added in 3.9
280 Name.starts_with("loadu.") || // Added in 3.9
281 Name.starts_with("lzcnt.") || // Added in 5.0
282 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
283 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("movddup") || // Added in 3.9
285 Name.starts_with("move.s") || // Added in 4.0
286 Name.starts_with("movshdup") || // Added in 3.9
287 Name.starts_with("movsldup") || // Added in 3.9
288 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
289 Name.starts_with("or.") || // Added in 3.9
290 Name.starts_with("pabs.") || // Added in 6.0
291 Name.starts_with("packssdw.") || // Added in 5.0
292 Name.starts_with("packsswb.") || // Added in 5.0
293 Name.starts_with("packusdw.") || // Added in 5.0
294 Name.starts_with("packuswb.") || // Added in 5.0
295 Name.starts_with("padd.") || // Added in 4.0
296 Name.starts_with("padds.") || // Added in 8.0
297 Name.starts_with("paddus.") || // Added in 8.0
298 Name.starts_with("palignr.") || // Added in 3.9
299 Name.starts_with("pand.") || // Added in 3.9
300 Name.starts_with("pandn.") || // Added in 3.9
301 Name.starts_with("pavg") || // Added in 6.0
302 Name.starts_with("pbroadcast") || // Added in 6.0
303 Name.starts_with("pcmpeq.") || // Added in 3.9
304 Name.starts_with("pcmpgt.") || // Added in 3.9
305 Name.starts_with("perm.df.") || // Added in 3.9
306 Name.starts_with("perm.di.") || // Added in 3.9
307 Name.starts_with("permvar.") || // Added in 7.0
308 Name.starts_with("pmaddubs.w.") || // Added in 7.0
309 Name.starts_with("pmaddw.d.") || // Added in 7.0
310 Name.starts_with("pmax") || // Added in 4.0
311 Name.starts_with("pmin") || // Added in 4.0
312 Name == "pmov.qd.256" || // Added in 9.0
313 Name == "pmov.qd.512" || // Added in 9.0
314 Name == "pmov.wb.256" || // Added in 9.0
315 Name == "pmov.wb.512" || // Added in 9.0
316 Name.starts_with("pmovsx") || // Added in 4.0
317 Name.starts_with("pmovzx") || // Added in 4.0
318 Name.starts_with("pmul.dq.") || // Added in 4.0
319 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
320 Name.starts_with("pmulh.w.") || // Added in 7.0
321 Name.starts_with("pmulhu.w.") || // Added in 7.0
322 Name.starts_with("pmull.") || // Added in 4.0
323 Name.starts_with("pmultishift.qb.") || // Added in 8.0
324 Name.starts_with("pmulu.dq.") || // Added in 4.0
325 Name.starts_with("por.") || // Added in 3.9
326 Name.starts_with("prol.") || // Added in 8.0
327 Name.starts_with("prolv.") || // Added in 8.0
328 Name.starts_with("pror.") || // Added in 8.0
329 Name.starts_with("prorv.") || // Added in 8.0
330 Name.starts_with("pshuf.b.") || // Added in 4.0
331 Name.starts_with("pshuf.d.") || // Added in 3.9
332 Name.starts_with("pshufh.w.") || // Added in 3.9
333 Name.starts_with("pshufl.w.") || // Added in 3.9
334 Name.starts_with("psll.d") || // Added in 4.0
335 Name.starts_with("psll.q") || // Added in 4.0
336 Name.starts_with("psll.w") || // Added in 4.0
337 Name.starts_with("pslli") || // Added in 4.0
338 Name.starts_with("psllv") || // Added in 4.0
339 Name.starts_with("psra.d") || // Added in 4.0
340 Name.starts_with("psra.q") || // Added in 4.0
341 Name.starts_with("psra.w") || // Added in 4.0
342 Name.starts_with("psrai") || // Added in 4.0
343 Name.starts_with("psrav") || // Added in 4.0
344 Name.starts_with("psrl.d") || // Added in 4.0
345 Name.starts_with("psrl.q") || // Added in 4.0
346 Name.starts_with("psrl.w") || // Added in 4.0
347 Name.starts_with("psrli") || // Added in 4.0
348 Name.starts_with("psrlv") || // Added in 4.0
349 Name.starts_with("psub.") || // Added in 4.0
350 Name.starts_with("psubs.") || // Added in 8.0
351 Name.starts_with("psubus.") || // Added in 8.0
352 Name.starts_with("pternlog.") || // Added in 7.0
353 Name.starts_with("punpckh") || // Added in 3.9
354 Name.starts_with("punpckl") || // Added in 3.9
355 Name.starts_with("pxor.") || // Added in 3.9
356 Name.starts_with("shuf.f") || // Added in 6.0
357 Name.starts_with("shuf.i") || // Added in 6.0
358 Name.starts_with("shuf.p") || // Added in 4.0
359 Name.starts_with("sqrt.p") || // Added in 7.0
360 Name.starts_with("store.b.") || // Added in 3.9
361 Name.starts_with("store.d.") || // Added in 3.9
362 Name.starts_with("store.p") || // Added in 3.9
363 Name.starts_with("store.q.") || // Added in 3.9
364 Name.starts_with("store.w.") || // Added in 3.9
365 Name == "store.ss" || // Added in 7.0
366 Name.starts_with("storeu.") || // Added in 3.9
367 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
368 Name.starts_with("ucmp.") || // Added in 5.0
369 Name.starts_with("unpckh.") || // Added in 3.9
370 Name.starts_with("unpckl.") || // Added in 3.9
371 Name.starts_with("valign.") || // Added in 4.0
372 Name == "vcvtph2ps.128" || // Added in 11.0
373 Name == "vcvtph2ps.256" || // Added in 11.0
374 Name.starts_with("vextract") || // Added in 4.0
375 Name.starts_with("vfmadd.") || // Added in 7.0
376 Name.starts_with("vfmaddsub.") || // Added in 7.0
377 Name.starts_with("vfnmadd.") || // Added in 7.0
378 Name.starts_with("vfnmsub.") || // Added in 7.0
379 Name.starts_with("vpdpbusd.") || // Added in 7.0
380 Name.starts_with("vpdpbusds.") || // Added in 7.0
381 Name.starts_with("vpdpwssd.") || // Added in 7.0
382 Name.starts_with("vpdpwssds.") || // Added in 7.0
383 Name.starts_with("vpermi2var.") || // Added in 7.0
384 Name.starts_with("vpermil.p") || // Added in 3.9
385 Name.starts_with("vpermilvar.") || // Added in 4.0
386 Name.starts_with("vpermt2var.") || // Added in 7.0
387 Name.starts_with("vpmadd52") || // Added in 7.0
388 Name.starts_with("vpshld.") || // Added in 7.0
389 Name.starts_with("vpshldv.") || // Added in 8.0
390 Name.starts_with("vpshrd.") || // Added in 7.0
391 Name.starts_with("vpshrdv.") || // Added in 8.0
392 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
393 Name.starts_with("xor.")); // Added in 3.9
394
395 if (Name.consume_front("mask3."))
396 // 'avx512.mask3.*'
397 return (Name.starts_with("vfmadd.") || // Added in 7.0
398 Name.starts_with("vfmaddsub.") || // Added in 7.0
399 Name.starts_with("vfmsub.") || // Added in 7.0
400 Name.starts_with("vfmsubadd.") || // Added in 7.0
401 Name.starts_with("vfnmsub.")); // Added in 7.0
402
403 if (Name.consume_front("maskz."))
404 // 'avx512.maskz.*'
405 return (Name.starts_with("pternlog.") || // Added in 7.0
406 Name.starts_with("vfmadd.") || // Added in 7.0
407 Name.starts_with("vfmaddsub.") || // Added in 7.0
408 Name.starts_with("vpdpbusd.") || // Added in 7.0
409 Name.starts_with("vpdpbusds.") || // Added in 7.0
410 Name.starts_with("vpdpwssd.") || // Added in 7.0
411 Name.starts_with("vpdpwssds.") || // Added in 7.0
412 Name.starts_with("vpermt2var.") || // Added in 7.0
413 Name.starts_with("vpmadd52") || // Added in 7.0
414 Name.starts_with("vpshldv.") || // Added in 8.0
415 Name.starts_with("vpshrdv.")); // Added in 8.0
416
417 // 'avx512.*'
418 return (Name == "movntdqa" || // Added in 5.0
419 Name == "pmul.dq.512" || // Added in 7.0
420 Name == "pmulu.dq.512" || // Added in 7.0
421 Name.starts_with("broadcastm") || // Added in 6.0
422 Name.starts_with("cmp.p") || // Added in 12.0
423 Name.starts_with("cvtb2mask.") || // Added in 7.0
424 Name.starts_with("cvtd2mask.") || // Added in 7.0
425 Name.starts_with("cvtmask2") || // Added in 5.0
426 Name.starts_with("cvtq2mask.") || // Added in 7.0
427 Name == "cvtusi2sd" || // Added in 7.0
428 Name.starts_with("cvtw2mask.") || // Added in 7.0
429 Name == "kand.w" || // Added in 7.0
430 Name == "kandn.w" || // Added in 7.0
431 Name == "knot.w" || // Added in 7.0
432 Name == "kor.w" || // Added in 7.0
433 Name == "kortestc.w" || // Added in 7.0
434 Name == "kortestz.w" || // Added in 7.0
435 Name.starts_with("kunpck") || // added in 6.0
436 Name == "kxnor.w" || // Added in 7.0
437 Name == "kxor.w" || // Added in 7.0
438 Name.starts_with("padds.") || // Added in 8.0
439 Name.starts_with("pbroadcast") || // Added in 3.9
440 Name.starts_with("prol") || // Added in 8.0
441 Name.starts_with("pror") || // Added in 8.0
442 Name.starts_with("psll.dq") || // Added in 3.9
443 Name.starts_with("psrl.dq") || // Added in 3.9
444 Name.starts_with("psubs.") || // Added in 8.0
445 Name.starts_with("ptestm") || // Added in 6.0
446 Name.starts_with("ptestnm") || // Added in 6.0
447 Name.starts_with("storent.") || // Added in 3.9
448 Name.starts_with("vbroadcast.s") || // Added in 7.0
449 Name.starts_with("vpshld.") || // Added in 8.0
450 Name.starts_with("vpshrd.")); // Added in 8.0
451 }
452
453 if (Name.consume_front("fma."))
454 return (Name.starts_with("vfmadd.") || // Added in 7.0
455 Name.starts_with("vfmsub.") || // Added in 7.0
456 Name.starts_with("vfmsubadd.") || // Added in 7.0
457 Name.starts_with("vfnmadd.") || // Added in 7.0
458 Name.starts_with("vfnmsub.")); // Added in 7.0
459
460 if (Name.consume_front("fma4."))
461 return Name.starts_with("vfmadd.s"); // Added in 7.0
462
463 if (Name.consume_front("sse."))
464 return (Name == "add.ss" || // Added in 4.0
465 Name == "cvtsi2ss" || // Added in 7.0
466 Name == "cvtsi642ss" || // Added in 7.0
467 Name == "div.ss" || // Added in 4.0
468 Name == "mul.ss" || // Added in 4.0
469 Name.starts_with("sqrt.p") || // Added in 7.0
470 Name == "sqrt.ss" || // Added in 7.0
471 Name.starts_with("storeu.") || // Added in 3.9
472 Name == "sub.ss"); // Added in 4.0
473
474 if (Name.consume_front("sse2."))
475 return (Name == "add.sd" || // Added in 4.0
476 Name == "cvtdq2pd" || // Added in 3.9
477 Name == "cvtdq2ps" || // Added in 7.0
478 Name == "cvtps2pd" || // Added in 3.9
479 Name == "cvtsi2sd" || // Added in 7.0
480 Name == "cvtsi642sd" || // Added in 7.0
481 Name == "cvtss2sd" || // Added in 7.0
482 Name == "div.sd" || // Added in 4.0
483 Name == "mul.sd" || // Added in 4.0
484 Name.starts_with("padds.") || // Added in 8.0
485 Name.starts_with("paddus.") || // Added in 8.0
486 Name.starts_with("pcmpeq.") || // Added in 3.1
487 Name.starts_with("pcmpgt.") || // Added in 3.1
488 Name == "pmaxs.w" || // Added in 3.9
489 Name == "pmaxu.b" || // Added in 3.9
490 Name == "pmins.w" || // Added in 3.9
491 Name == "pminu.b" || // Added in 3.9
492 Name == "pmulu.dq" || // Added in 7.0
493 Name.starts_with("pshuf") || // Added in 3.9
494 Name.starts_with("psll.dq") || // Added in 3.7
495 Name.starts_with("psrl.dq") || // Added in 3.7
496 Name.starts_with("psubs.") || // Added in 8.0
497 Name.starts_with("psubus.") || // Added in 8.0
498 Name.starts_with("sqrt.p") || // Added in 7.0
499 Name == "sqrt.sd" || // Added in 7.0
500 Name == "storel.dq" || // Added in 3.9
501 Name.starts_with("storeu.") || // Added in 3.9
502 Name == "sub.sd"); // Added in 4.0
503
504 if (Name.consume_front("sse41."))
505 return (Name.starts_with("blendp") || // Added in 3.7
506 Name == "movntdqa" || // Added in 5.0
507 Name == "pblendw" || // Added in 3.7
508 Name == "pmaxsb" || // Added in 3.9
509 Name == "pmaxsd" || // Added in 3.9
510 Name == "pmaxud" || // Added in 3.9
511 Name == "pmaxuw" || // Added in 3.9
512 Name == "pminsb" || // Added in 3.9
513 Name == "pminsd" || // Added in 3.9
514 Name == "pminud" || // Added in 3.9
515 Name == "pminuw" || // Added in 3.9
516 Name.starts_with("pmovsx") || // Added in 3.8
517 Name.starts_with("pmovzx") || // Added in 3.9
518 Name == "pmuldq"); // Added in 7.0
519
520 if (Name.consume_front("sse42."))
521 return Name == "crc32.64.8"; // Added in 3.4
522
523 if (Name.consume_front("sse4a."))
524 return Name.starts_with("movnt."); // Added in 3.9
525
526 if (Name.consume_front("ssse3."))
527 return (Name == "pabs.b.128" || // Added in 6.0
528 Name == "pabs.d.128" || // Added in 6.0
529 Name == "pabs.w.128"); // Added in 6.0
530
531 if (Name.consume_front("xop."))
532 return (Name == "vpcmov" || // Added in 3.8
533 Name == "vpcmov.256" || // Added in 5.0
534 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
535 Name.starts_with("vprot")); // Added in 8.0
536
537 if (Name.consume_front("bmi."))
538 return (Name.starts_with("pdep.") || // Added in 23.0
539 Name.starts_with("pext.")); // Added in 23.0
540
541 return (Name == "addcarry.u32" || // Added in 8.0
542 Name == "addcarry.u64" || // Added in 8.0
543 Name == "addcarryx.u32" || // Added in 8.0
544 Name == "addcarryx.u64" || // Added in 8.0
545 Name == "subborrow.u32" || // Added in 8.0
546 Name == "subborrow.u64" || // Added in 8.0
547 Name.starts_with("vcvtph2ps.")); // Added in 11.0
548}
549
550static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
551 Function *&NewFn) {
552 // Only handle intrinsics that start with "x86.".
553 if (!Name.consume_front("x86."))
554 return false;
555
556 if (shouldUpgradeX86Intrinsic(F, Name)) {
557 NewFn = nullptr;
558 return true;
559 }
560
561 if (Name == "rdtscp") { // Added in 8.0
562 // If this intrinsic has 0 operands, it's the new version.
563 if (F->getFunctionType()->getNumParams() == 0)
564 return false;
565
566 rename(F);
567 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
568 Intrinsic::x86_rdtscp);
569 return true;
570 }
571
572 Intrinsic::ID ID;
573
574 // SSE4.1 ptest functions may have an old signature.
575 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
576 ID = StringSwitch<Intrinsic::ID>(Name)
577 .Case("c", Intrinsic::x86_sse41_ptestc)
578 .Case("z", Intrinsic::x86_sse41_ptestz)
579 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
580 .Default(Intrinsic::not_intrinsic);
581 if (ID != Intrinsic::not_intrinsic)
582 return upgradePTESTIntrinsic(F, ID, NewFn);
583
584 return false;
585 }
586
587 // Several blend and other instructions with masks used the wrong number of
588 // bits.
589
590 // Added in 3.6
591 ID = StringSwitch<Intrinsic::ID>(Name)
592 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
593 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
594 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
595 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
596 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
597 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
598 .Default(Intrinsic::not_intrinsic);
599 if (ID != Intrinsic::not_intrinsic)
600 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
601
602 if (Name.consume_front("avx512.")) {
603 if (Name.consume_front("mask.cmp.")) {
604 // Added in 7.0
605 ID = StringSwitch<Intrinsic::ID>(Name)
606 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
607 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
608 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
609 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
610 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
611 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
612 .Default(Intrinsic::not_intrinsic);
613 if (ID != Intrinsic::not_intrinsic)
614 return upgradeX86MaskedFPCompare(F, ID, NewFn);
615 } else if (Name.starts_with("vpdpbusd.") ||
616 Name.starts_with("vpdpbusds.")) {
617 // Added in 21.1
618 ID = StringSwitch<Intrinsic::ID>(Name)
619 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
620 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
621 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
622 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
623 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
624 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
625 .Default(Intrinsic::not_intrinsic);
626 if (ID != Intrinsic::not_intrinsic)
627 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
628 } else if (Name.starts_with("vpdpwssd.") ||
629 Name.starts_with("vpdpwssds.")) {
630 // Added in 21.1
631 ID = StringSwitch<Intrinsic::ID>(Name)
632 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
633 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
634 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
635 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
636 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
637 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
638 .Default(Intrinsic::not_intrinsic);
639 if (ID != Intrinsic::not_intrinsic)
640 return upgradeX86MultiplyAddWords(F, ID, NewFn);
641 }
642 return false; // No other 'x86.avx512.*'.
643 }
644
645 if (Name.consume_front("avx2.")) {
646 if (Name.consume_front("vpdpb")) {
647 // Added in 21.1
648 ID = StringSwitch<Intrinsic::ID>(Name)
649 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
650 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
651 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
652 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
653 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
654 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
655 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
656 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
657 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
658 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
659 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
660 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
661 .Default(Intrinsic::not_intrinsic);
662 if (ID != Intrinsic::not_intrinsic)
663 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
664 } else if (Name.consume_front("vpdpw")) {
665 // Added in 21.1
666 ID = StringSwitch<Intrinsic::ID>(Name)
667 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
668 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
669 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
670 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
671 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
672 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
673 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
674 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
675 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
676 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
677 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
678 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
679 .Default(Intrinsic::not_intrinsic);
680 if (ID != Intrinsic::not_intrinsic)
681 return upgradeX86MultiplyAddWords(F, ID, NewFn);
682 }
683 return false; // No other 'x86.avx2.*'
684 }
685
686 if (Name.consume_front("avx10.")) {
687 if (Name.consume_front("vpdpb")) {
688 // Added in 21.1
689 ID = StringSwitch<Intrinsic::ID>(Name)
690 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
691 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
692 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
693 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
694 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
695 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
696 .Default(Intrinsic::not_intrinsic);
697 if (ID != Intrinsic::not_intrinsic)
698 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
699 } else if (Name.consume_front("vpdpw")) {
700 ID = StringSwitch<Intrinsic::ID>(Name)
701 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
702 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
703 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
704 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
705 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
706 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
707 .Default(Intrinsic::not_intrinsic);
708 if (ID != Intrinsic::not_intrinsic)
709 return upgradeX86MultiplyAddWords(F, ID, NewFn);
710 }
711 return false; // No other 'x86.avx10.*'
712 }
713
714 if (Name.consume_front("avx512bf16.")) {
715 // Added in 9.0
716 ID = StringSwitch<Intrinsic::ID>(Name)
717 .Case("cvtne2ps2bf16.128",
718 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
719 .Case("cvtne2ps2bf16.256",
720 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
721 .Case("cvtne2ps2bf16.512",
722 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
723 .Case("mask.cvtneps2bf16.128",
724 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
725 .Case("cvtneps2bf16.256",
726 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
727 .Case("cvtneps2bf16.512",
728 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
729 .Default(Intrinsic::not_intrinsic);
730 if (ID != Intrinsic::not_intrinsic)
731 return upgradeX86BF16Intrinsic(F, ID, NewFn);
732
733 // Added in 9.0
734 ID = StringSwitch<Intrinsic::ID>(Name)
735 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
736 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
737 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
738 .Default(Intrinsic::not_intrinsic);
739 if (ID != Intrinsic::not_intrinsic)
740 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
741 return false; // No other 'x86.avx512bf16.*'.
742 }
743
744 if (Name.consume_front("xop.")) {
745 Intrinsic::ID ID = Intrinsic::not_intrinsic;
746 if (Name.starts_with("vpermil2")) { // Added in 3.9
747 // Upgrade any XOP PERMIL2 index operand still using a float/double
748 // vector.
749 auto Idx = F->getFunctionType()->getParamType(2);
750 if (Idx->isFPOrFPVectorTy()) {
751 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
752 unsigned EltSize = Idx->getScalarSizeInBits();
753 if (EltSize == 64 && IdxSize == 128)
754 ID = Intrinsic::x86_xop_vpermil2pd;
755 else if (EltSize == 32 && IdxSize == 128)
756 ID = Intrinsic::x86_xop_vpermil2ps;
757 else if (EltSize == 64 && IdxSize == 256)
758 ID = Intrinsic::x86_xop_vpermil2pd_256;
759 else
760 ID = Intrinsic::x86_xop_vpermil2ps_256;
761 }
762 } else if (F->arg_size() == 2)
763 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
764 ID = StringSwitch<Intrinsic::ID>(Name)
765 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
766 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
767 .Default(Intrinsic::not_intrinsic);
768
769 if (ID != Intrinsic::not_intrinsic) {
770 rename(F);
771 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
772 return true;
773 }
774 return false; // No other 'x86.xop.*'
775 }
776
777 if (Name == "seh.recoverfp") {
778 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
779 Intrinsic::eh_recoverfp);
780 return true;
781 }
782
783 return false;
784}
785
786// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
787// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
788static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
789 StringRef Name,
790 Function *&NewFn) {
791 if (Name.starts_with("rbit")) {
792 // '(arm|aarch64).rbit'.
793 NewFn = Intrinsic::getOrInsertDeclaration(
794 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
795 return true;
796 }
797
798 if (Name == "thread.pointer") {
799 // '(arm|aarch64).thread.pointer'.
800 NewFn = Intrinsic::getOrInsertDeclaration(
801 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
802 return true;
803 }
804
805 bool Neon = Name.consume_front("neon.");
806 if (Neon) {
807 // '(arm|aarch64).neon.*'.
808 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
809 // v16i8 respectively.
810 if (Name.consume_front("bfdot.")) {
811 // (arm|aarch64).neon.bfdot.*'.
812 Intrinsic::ID ID =
813 StringSwitch<Intrinsic::ID>(Name)
814 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
815 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
816 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
817 .Default(Intrinsic::not_intrinsic);
818 if (ID != Intrinsic::not_intrinsic) {
819 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
820 assert((OperandWidth == 64 || OperandWidth == 128) &&
821 "Unexpected operand width");
822 LLVMContext &Ctx = F->getParent()->getContext();
823 std::array<Type *, 2> Tys{
824 {F->getReturnType(),
825 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
826 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
827 return true;
828 }
829 return false; // No other '(arm|aarch64).neon.bfdot.*'.
830 }
831
832 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
833 // anymore and accept v8bf16 instead of v16i8.
834 if (Name.consume_front("bfm")) {
835 // (arm|aarch64).neon.bfm*'.
836 if (Name.consume_back(".v4f32.v16i8")) {
837 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
838 Intrinsic::ID ID =
839 StringSwitch<Intrinsic::ID>(Name)
840 .Case("mla",
841 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
842 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
843 .Case("lalb",
844 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
845 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
846 .Case("lalt",
847 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
849 .Default(Intrinsic::not_intrinsic);
850 if (ID != Intrinsic::not_intrinsic) {
851 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
852 return true;
853 }
854 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
855 }
856 return false; // No other '(arm|aarch64).neon.bfm*.
857 }
858 // Continue on to Aarch64 Neon or Arm Neon.
859 }
860 // Continue on to Arm or Aarch64.
861
862 if (IsArm) {
863 // 'arm.*'.
864 if (Neon) {
865 // 'arm.neon.*'.
866 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
867 .StartsWith("vclz.", Intrinsic::ctlz)
868 .StartsWith("vcnt.", Intrinsic::ctpop)
869 .StartsWith("vqadds.", Intrinsic::sadd_sat)
870 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
871 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
872 .StartsWith("vqsubu.", Intrinsic::usub_sat)
873 .StartsWith("vrinta.", Intrinsic::round)
874 .StartsWith("vrintn.", Intrinsic::roundeven)
875 .StartsWith("vrintm.", Intrinsic::floor)
876 .StartsWith("vrintp.", Intrinsic::ceil)
877 .StartsWith("vrintx.", Intrinsic::rint)
878 .StartsWith("vrintz.", Intrinsic::trunc)
879 .Default(Intrinsic::not_intrinsic);
880 if (ID != Intrinsic::not_intrinsic) {
881 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
882 F->arg_begin()->getType());
883 return true;
884 }
885
886 if (Name.consume_front("vst")) {
887 // 'arm.neon.vst*'.
888 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
889 SmallVector<StringRef, 2> Groups;
890 if (vstRegex.match(Name, &Groups)) {
891 static const Intrinsic::ID StoreInts[] = {
892 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
893 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
894
895 static const Intrinsic::ID StoreLaneInts[] = {
896 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
897 Intrinsic::arm_neon_vst4lane};
898
899 auto fArgs = F->getFunctionType()->params();
900 Type *Tys[] = {fArgs[0], fArgs[1]};
901 if (Groups[1].size() == 1)
902 NewFn = Intrinsic::getOrInsertDeclaration(
903 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
904 else
905 NewFn = Intrinsic::getOrInsertDeclaration(
906 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
907 return true;
908 }
909 return false; // No other 'arm.neon.vst*'.
910 }
911
912 return false; // No other 'arm.neon.*'.
913 }
914
915 if (Name.consume_front("mve.")) {
916 // 'arm.mve.*'.
917 if (Name == "vctp64") {
918 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
919 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
920 // the function and deal with it below in UpgradeIntrinsicCall.
921 rename(F);
922 return true;
923 }
924 return false; // Not 'arm.mve.vctp64'.
925 }
926
927 if (Name.starts_with("vrintn.v")) {
928 NewFn = Intrinsic::getOrInsertDeclaration(
929 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
930 return true;
931 }
932
933 // These too are changed to accept a v2i1 instead of the old v4i1.
934 if (Name.consume_back(".v4i1")) {
935 // 'arm.mve.*.v4i1'.
936 if (Name.consume_back(".predicated.v2i64.v4i32"))
937 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
938 return Name == "mull.int" || Name == "vqdmull";
939
940 if (Name.consume_back(".v2i64")) {
941 // 'arm.mve.*.v2i64.v4i1'
942 bool IsGather = Name.consume_front("vldr.gather.");
943 if (IsGather || Name.consume_front("vstr.scatter.")) {
944 if (Name.consume_front("base.")) {
945 // Optional 'wb.' prefix.
946 Name.consume_front("wb.");
947 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
948 // predicated.v2i64.v2i64.v4i1'.
949 return Name == "predicated.v2i64";
950 }
951
952 if (Name.consume_front("offset.predicated."))
953 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
954 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
955
956 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
957 return false;
958 }
959
960 return false; // No other 'arm.mve.*.v2i64.v4i1'.
961 }
962 return false; // No other 'arm.mve.*.v4i1'.
963 }
964 return false; // No other 'arm.mve.*'.
965 }
966
967 if (Name.consume_front("cde.vcx")) {
968 // 'arm.cde.vcx*'.
969 if (Name.consume_back(".predicated.v2i64.v4i1"))
970 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
971 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
972 Name == "3q" || Name == "3qa";
973
974 return false; // No other 'arm.cde.vcx*'.
975 }
976 } else {
977 // 'aarch64.*'.
978 if (Neon) {
979 // 'aarch64.neon.*'.
980 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
981 .StartsWith("frintn", Intrinsic::roundeven)
982 .StartsWith("rbit", Intrinsic::bitreverse)
983 .Default(Intrinsic::not_intrinsic);
984 if (ID != Intrinsic::not_intrinsic) {
985 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
986 F->arg_begin()->getType());
987 return true;
988 }
989
990 if (Name.starts_with("addp")) {
991 // 'aarch64.neon.addp*'.
992 if (F->arg_size() != 2)
993 return false; // Invalid IR.
994 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
995 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
996 NewFn = Intrinsic::getOrInsertDeclaration(
997 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
998 return true;
999 }
1000 }
1001
1002 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1003 if (Name.starts_with("bfcvt")) {
1004 NewFn = nullptr;
1005 return true;
1006 }
1007
1008 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1009 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1010 NewFn = nullptr;
1011 return true;
1012 }
1013
1014 return false; // No other 'aarch64.neon.*'.
1015 }
1016 if (Name.consume_front("sve.")) {
1017 // 'aarch64.sve.*'.
1018 if (Name.consume_front("bf")) {
1019 if (Name == "mmla") {
1020 Type *Tys[] = {F->getReturnType(),
1021 std::next(F->arg_begin())->getType()};
1022 NewFn = Intrinsic::getOrInsertDeclaration(
1023 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1024 return true;
1025 }
1026 if (Name.consume_back(".lane")) {
1027 // 'aarch64.sve.bf*.lane'.
1028 Intrinsic::ID ID =
1029 StringSwitch<Intrinsic::ID>(Name)
1030 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1031 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1032 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1033 .Default(Intrinsic::not_intrinsic);
1034 if (ID != Intrinsic::not_intrinsic) {
1035 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1036 return true;
1037 }
1038 return false; // No other 'aarch64.sve.bf*.lane'.
1039 }
1040 return false; // No other 'aarch64.sve.bf*'.
1041 }
1042
1043 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1044 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1045 NewFn = nullptr;
1046 return true;
1047 }
1048
1049 if (Name.consume_front("addqv")) {
1050 // 'aarch64.sve.addqv'.
1051 if (!F->getReturnType()->isFPOrFPVectorTy())
1052 return false;
1053
1054 auto Args = F->getFunctionType()->params();
1055 Type *Tys[] = {F->getReturnType(), Args[1]};
1056 NewFn = Intrinsic::getOrInsertDeclaration(
1057 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("ld")) {
1062 // 'aarch64.sve.ld*'.
1063 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1064 if (LdRegex.match(Name)) {
1065 Type *ScalarTy =
1066 cast<VectorType>(F->getReturnType())->getElementType();
1067 ElementCount EC =
1068 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1069 assert(F->arg_size() == 2 &&
1070 "Expected 2 arguments for ld* intrinsic.");
1071 Type *PtrTy = F->getArg(1)->getType();
1072 Type *Ty = VectorType::get(ScalarTy, EC);
1073 static const Intrinsic::ID LoadIDs[] = {
1074 Intrinsic::aarch64_sve_ld2_sret,
1075 Intrinsic::aarch64_sve_ld3_sret,
1076 Intrinsic::aarch64_sve_ld4_sret,
1077 };
1078 NewFn = Intrinsic::getOrInsertDeclaration(
1079 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1080 return true;
1081 }
1082 return false; // No other 'aarch64.sve.ld*'.
1083 }
1084
1085 if (Name.consume_front("tuple.")) {
1086 // 'aarch64.sve.tuple.*'.
1087 if (Name.starts_with("get")) {
1088 // 'aarch64.sve.tuple.get*'.
1089 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1090 NewFn = Intrinsic::getOrInsertDeclaration(
1091 F->getParent(), Intrinsic::vector_extract, Tys);
1092 return true;
1093 }
1094
1095 if (Name.starts_with("set")) {
1096 // 'aarch64.sve.tuple.set*'.
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {Args[0], Args[2], Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 F->getParent(), Intrinsic::vector_insert, Tys);
1101 return true;
1102 }
1103
1104 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1105 if (CreateTupleRegex.match(Name)) {
1106 // 'aarch64.sve.tuple.create*'.
1107 auto Args = F->getFunctionType()->params();
1108 Type *Tys[] = {F->getReturnType(), Args[1]};
1109 NewFn = Intrinsic::getOrInsertDeclaration(
1110 F->getParent(), Intrinsic::vector_insert, Tys);
1111 return true;
1112 }
1113 return false; // No other 'aarch64.sve.tuple.*'.
1114 }
1115
1116 if (Name.starts_with("rev.nxv")) {
1117 // 'aarch64.sve.rev.<Ty>'
1118 NewFn = Intrinsic::getOrInsertDeclaration(
1119 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1120 return true;
1121 }
1122
1123 return false; // No other 'aarch64.sve.*'.
1124 }
1125 if (Name.consume_front("sme.")) {
1126 // 'aarch64.sme.*'.
1127 if (Name.consume_front("ftmopa.")) {
1128 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1129 // intrinsics to model their FPMR dependency.
1130 Intrinsic::ID ID =
1131 StringSwitch<Intrinsic::ID>(Name)
1132 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1133 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1134 .Default(Intrinsic::not_intrinsic);
1135 if (ID != Intrinsic::not_intrinsic) {
1136 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1137 return true;
1138 }
1139 return false; // No other 'aarch64.sme.ftmopa.*'.
1140 }
1141
1142 return false; // No other 'aarch64.sme.*'.
1143 }
1144 }
1145 return false; // No other 'arm.*', 'aarch64.*'.
1146}
1147
1148static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1149 StringRef Name) {
1150 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1151 Intrinsic::ID ID =
1152 StringSwitch<Intrinsic::ID>(Name)
1153 .Case("im2col.3d",
1154 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1155 .Case("im2col.4d",
1156 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1157 .Case("im2col.5d",
1158 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1159 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1160 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1161 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1162 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1163 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1164 .Default(Intrinsic::not_intrinsic);
1165
1166 if (ID == Intrinsic::not_intrinsic)
1167 return ID;
1168
1169 // These intrinsics may need upgrade for two reasons:
1170 // (1) When the address-space of the first argument is shared[AS=3]
1171 // (and we upgrade it to use shared_cluster address-space[AS=7])
1172 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1173 NVPTXAS::ADDRESS_SPACE_SHARED)
1174 return ID;
1175
1176 // (2) When there are only two boolean flag arguments at the end:
1177 //
1178 // The last three parameters of the older version of these
1179 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1180 //
1181 // The newer version reads as:
1182 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1183 //
1184 // So, when the type of the [N-3]rd argument is "not i1", then
1185 // it is the older version and we need to upgrade.
1186 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1187 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1188 if (!ArgType->isIntegerTy(1))
1189 return ID;
1190 }
1191
1192 return Intrinsic::not_intrinsic;
1193}
1194
1195static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1196 StringRef Name) {
1197 if (Name.consume_front("mapa.shared.cluster"))
1198 if (F->getReturnType()->getPointerAddressSpace() ==
1199 NVPTXAS::ADDRESS_SPACE_SHARED)
1200 return Intrinsic::nvvm_mapa_shared_cluster;
1201
1202 if (Name.consume_front("cp.async.bulk.")) {
1203 Intrinsic::ID ID =
1204 StringSwitch<Intrinsic::ID>(Name)
1205 .Case("global.to.shared.cluster",
1206 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1207 .Case("shared.cta.to.cluster",
1208 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1209 .Default(Intrinsic::not_intrinsic);
1210
1211 if (ID != Intrinsic::not_intrinsic)
1212 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1213 NVPTXAS::ADDRESS_SPACE_SHARED)
1214 return ID;
1215 }
1216
1217 return Intrinsic::not_intrinsic;
1218}
1219
1220static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1221 if (Name.consume_front("fma.rn."))
1222 return StringSwitch<Intrinsic::ID>(Name)
1223 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1224 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1225 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1226 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1227 .Default(Intrinsic::not_intrinsic);
1228
1229 if (Name.consume_front("fmax."))
1230 return StringSwitch<Intrinsic::ID>(Name)
1231 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1232 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1233 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1234 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1235 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1236 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1237 .Case("ftz.nan.xorsign.abs.bf16",
1238 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1239 .Case("ftz.nan.xorsign.abs.bf16x2",
1240 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1241 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1242 .Case("ftz.xorsign.abs.bf16x2",
1243 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1244 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1245 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1246 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1247 .Case("nan.xorsign.abs.bf16x2",
1248 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1249 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1250 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1251 .Default(Intrinsic::not_intrinsic);
1252
1253 if (Name.consume_front("fmin."))
1254 return StringSwitch<Intrinsic::ID>(Name)
1255 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1256 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1257 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1258 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1259 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1260 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1261 .Case("ftz.nan.xorsign.abs.bf16",
1262 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1263 .Case("ftz.nan.xorsign.abs.bf16x2",
1264 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1265 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1266 .Case("ftz.xorsign.abs.bf16x2",
1267 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1268 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1269 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1270 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1271 .Case("nan.xorsign.abs.bf16x2",
1272 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1273 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1274 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1275 .Default(Intrinsic::not_intrinsic);
1276
1277 if (Name.consume_front("neg."))
1278 return StringSwitch<Intrinsic::ID>(Name)
1279 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1280 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1281 .Default(Intrinsic::not_intrinsic);
1282
1283 return Intrinsic::not_intrinsic;
1284}
1285
1286static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1287 return Name.consume_front("local") || Name.consume_front("shared") ||
1288 Name.consume_front("global") || Name.consume_front("constant") ||
1289 Name.consume_front("param");
1290}
1291
1292static bool convertIntrinsicValidType(StringRef Name,
1293 const FunctionType *FuncTy) {
1294 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1295 if (Name.starts_with("to.fp16")) {
1296 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1297 HalfTy) &&
1298 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1299 FuncTy->getReturnType());
1300 }
1301
1302 if (Name.starts_with("from.fp16")) {
1303 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1304 HalfTy) &&
1305 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1306 FuncTy->getReturnType());
1307 }
1308
1309 return false;
1310}
1311
1312static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1313 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1314 if (IID == Intrinsic::not_intrinsic)
1315 return false;
1316
1317 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1318 if (Defaults.empty())
1319 return false;
1320
1321 // Overloaded intrinsics are out of scope for the default-arg feature
1322 // and will be supported in a follow-up.
1323 if (Intrinsic::isOverloaded(IID))
1324 return false;
1325
1326 // Get the canonical full declaration for this intrinsic.
1327 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1328
1329 // If the existing declaration already has all args, nothing to upgrade
1330 if (F->arg_size() >= FullDecl->arg_size())
1331 return false;
1332
1333 // Defaults are a contiguous trailing block, so checking the first missing
1334 // argument is enough.
1335 if (F->arg_size() < FirstDefault)
1336 return false;
1337
1338 NewFn = FullDecl;
1339 return true;
1340}
1341
1342static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1343 bool CanUpgradeDebugIntrinsicsToRecords) {
1344 assert(F && "Illegal to upgrade a non-existent Function.");
1345
1346 StringRef Name = F->getName();
1347
1348 // Quickly eliminate it, if it's not a candidate.
1349 if (!Name.consume_front("llvm.") || Name.empty())
1350 return false;
1351
1352 switch (Name[0]) {
1353 default: break;
1354 case 'a': {
1355 bool IsArm = Name.consume_front("arm.");
1356 if (IsArm || Name.consume_front("aarch64.")) {
1357 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1358 return true;
1359 break;
1360 }
1361
1362 if (Name.consume_front("amdgcn.")) {
1363 if (Name == "alignbit") {
1364 // Target specific intrinsic became redundant
1365 NewFn = Intrinsic::getOrInsertDeclaration(
1366 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1367 return true;
1368 }
1369
1370 if (Name.consume_front("atomic.")) {
1371 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1372 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1373 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1374 // and usub_sat so there's no new declaration.
1375 NewFn = nullptr;
1376 return true;
1377 }
1378 break; // No other 'amdgcn.atomic.*'
1379 }
1380
1381 switch (F->getIntrinsicID()) {
1382 default:
1383 break;
1384 // Legacy wmma iu intrinsics without the optional clamp operand.
1385 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1386 if (F->arg_size() == 7) {
1387 NewFn = nullptr;
1388 return true;
1389 }
1390 break;
1391 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1392 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1393 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1394 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1395 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1396 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1397 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1398 if (F->arg_size() == 8) {
1399 NewFn = nullptr;
1400 return true;
1401 }
1402 break;
1403 }
1404
1405 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1406 Name.consume_front("flat.atomic.")) {
1407 if (Name.starts_with("fadd") ||
1408 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1409 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1410 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1411 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1412 // declaration.
1413 NewFn = nullptr;
1414 return true;
1415 }
1416 }
1417
1418 if (Name.starts_with("ldexp.")) {
1419 // Target specific intrinsic became redundant
1420 NewFn = Intrinsic::getOrInsertDeclaration(
1421 F->getParent(), Intrinsic::ldexp,
1422 {F->getReturnType(), F->getArg(1)->getType()});
1423 return true;
1424 }
1425 break; // No other 'amdgcn.*'
1426 }
1427
1428 break;
1429 }
1430 case 'c': {
1431 if (F->arg_size() == 1) {
1432 if (Name.consume_front("convert.")) {
1433 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1434 NewFn = nullptr;
1435 return true;
1436 }
1437 }
1438
1439 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1440 .StartsWith("ctlz.", Intrinsic::ctlz)
1441 .StartsWith("cttz.", Intrinsic::cttz)
1442 .Default(Intrinsic::not_intrinsic);
1443 if (ID != Intrinsic::not_intrinsic) {
1444 rename(F);
1445 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1446 F->arg_begin()->getType());
1447 return true;
1448 }
1449 }
1450
1451 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1452 if (Name == "coro.end" &&
1453 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1454 CoroEndID = Intrinsic::coro_end;
1455 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1456 CoroEndID = Intrinsic::coro_end_async;
1457
1458 if (CoroEndID != Intrinsic::not_intrinsic) {
1459 rename(F);
1460 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1461 return true;
1462 }
1463
1464 break;
1465 }
1466 case 'd':
1467 if (Name.consume_front("dbg.")) {
1468 // Mark debug intrinsics for upgrade to new debug format.
1469 if (CanUpgradeDebugIntrinsicsToRecords) {
1470 if (Name == "addr" || Name == "value" || Name == "assign" ||
1471 Name == "declare" || Name == "label") {
1472 // There's no function to replace these with.
1473 NewFn = nullptr;
1474 // But we do want these to get upgraded.
1475 return true;
1476 }
1477 }
1478 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1479 // converted to DbgVariableRecords later.
1480 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1481 rename(F);
1482 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1483 Intrinsic::dbg_value);
1484 return true;
1485 }
1486 break; // No other 'dbg.*'.
1487 }
1488 break;
1489 case 'e':
1490 if (Name.consume_front("experimental.vector.")) {
1491 Intrinsic::ID ID =
1492 StringSwitch<Intrinsic::ID>(Name)
1493 // Skip over extract.last.active, otherwise it will be 'upgraded'
1494 // to a regular vector extract which is a different operation.
1495 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1496 .StartsWith("extract.", Intrinsic::vector_extract)
1497 .StartsWith("insert.", Intrinsic::vector_insert)
1498 .StartsWith("reverse.", Intrinsic::vector_reverse)
1499 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1500 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1501 .StartsWith("partial.reduce.add",
1502 Intrinsic::vector_partial_reduce_add)
1503 .Default(Intrinsic::not_intrinsic);
1504 if (ID != Intrinsic::not_intrinsic) {
1505 const auto *FT = F->getFunctionType();
1506 SmallVector<Type *, 2> Tys;
1507 if (ID == Intrinsic::vector_extract ||
1508 ID == Intrinsic::vector_interleave2)
1509 // Extracting overloads the return type.
1510 Tys.push_back(FT->getReturnType());
1511 if (ID != Intrinsic::vector_interleave2)
1512 Tys.push_back(FT->getParamType(0));
1513 if (ID == Intrinsic::vector_insert ||
1514 ID == Intrinsic::vector_partial_reduce_add)
1515 // Inserting overloads the inserted type.
1516 Tys.push_back(FT->getParamType(1));
1517 rename(F);
1518 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1519 return true;
1520 }
1521
1522 if (Name.consume_front("reduce.")) {
1523 SmallVector<StringRef, 2> Groups;
1524 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1525 if (R.match(Name, &Groups))
1526 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1527 .Case("add", Intrinsic::vector_reduce_add)
1528 .Case("mul", Intrinsic::vector_reduce_mul)
1529 .Case("and", Intrinsic::vector_reduce_and)
1530 .Case("or", Intrinsic::vector_reduce_or)
1531 .Case("xor", Intrinsic::vector_reduce_xor)
1532 .Case("smax", Intrinsic::vector_reduce_smax)
1533 .Case("smin", Intrinsic::vector_reduce_smin)
1534 .Case("umax", Intrinsic::vector_reduce_umax)
1535 .Case("umin", Intrinsic::vector_reduce_umin)
1536 .Case("fmax", Intrinsic::vector_reduce_fmax)
1537 .Case("fmin", Intrinsic::vector_reduce_fmin)
1538 .Default(Intrinsic::not_intrinsic);
1539
1540 bool V2 = false;
1541 if (ID == Intrinsic::not_intrinsic) {
1542 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1543 Groups.clear();
1544 V2 = true;
1545 if (R2.match(Name, &Groups))
1546 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1547 .Case("fadd", Intrinsic::vector_reduce_fadd)
1548 .Case("fmul", Intrinsic::vector_reduce_fmul)
1549 .Default(Intrinsic::not_intrinsic);
1550 }
1551 if (ID != Intrinsic::not_intrinsic) {
1552 rename(F);
1553 auto Args = F->getFunctionType()->params();
1554 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1555 {Args[V2 ? 1 : 0]});
1556 return true;
1557 }
1558 break; // No other 'expermental.vector.reduce.*'.
1559 }
1560
1561 if (Name.consume_front("splice"))
1562 return true;
1563 break; // No other 'experimental.vector.*'.
1564 }
1565 if (Name.consume_front("experimental.stepvector.")) {
1566 Intrinsic::ID ID = Intrinsic::stepvector;
1567 rename(F);
1568 NewFn = Intrinsic::getOrInsertDeclaration(
1569 F->getParent(), ID, F->getFunctionType()->getReturnType());
1570 return true;
1571 }
1572 break; // No other 'e*'.
1573 case 'f':
1574 if (Name.starts_with("flt.rounds")) {
1575 rename(F);
1576 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1577 Intrinsic::get_rounding);
1578 return true;
1579 }
1580 break;
1581 case 'i':
1582 if (Name.starts_with("invariant.group.barrier")) {
1583 // Rename invariant.group.barrier to launder.invariant.group
1584 auto Args = F->getFunctionType()->params();
1585 Type* ObjectPtr[1] = {Args[0]};
1586 rename(F);
1587 NewFn = Intrinsic::getOrInsertDeclaration(
1588 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1589 return true;
1590 }
1591 break;
1592 case 'l': {
1593 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1594 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1595 if (IsLifetimeStart || IsLifetimeEnd) {
1596 if (F->arg_size() == 2) {
1597 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1598 : Intrinsic::lifetime_end;
1599 rename(F);
1600 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1601 // arguments. Use the Ptr argument to create new declaration.
1602 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1603 F->getArg(1)->getType());
1604 return true;
1605 } else if (F->arg_size() == 1 && Name == ".i64") {
1606 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1607 // Autoupgrade prior to
1608 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1609 // intrinsic with no expected calls. To allow auto-upgrade process to
1610 // delete such invalid intrinsic declaration, set NewFn = nullptr
1611 // and return true here. If there are actual calls to this intrinsic
1612 // (which is not expected), they will be deleted in
1613 // UpgradeIntrinsicCall.
1614 NewFn = nullptr;
1615 return true;
1616 }
1617 }
1618 break;
1619 }
1620 case 'm': {
1621 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1622 // alignment parameter to embedding the alignment as an attribute of
1623 // the pointer args.
1624 if (unsigned ID = StringSwitch<unsigned>(Name)
1625 .StartsWith("memcpy.", Intrinsic::memcpy)
1626 .StartsWith("memmove.", Intrinsic::memmove)
1627 .Default(0)) {
1628 if (F->arg_size() == 5) {
1629 rename(F);
1630 // Get the types of dest, src, and len
1631 ArrayRef<Type *> ParamTypes =
1632 F->getFunctionType()->params().slice(0, 3);
1633 NewFn =
1634 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1635 return true;
1636 }
1637 }
1638 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1639 rename(F);
1640 // Get the types of dest, and len
1641 const auto *FT = F->getFunctionType();
1642 Type *ParamTypes[2] = {
1643 FT->getParamType(0), // Dest
1644 FT->getParamType(2) // len
1645 };
1646 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1647 Intrinsic::memset, ParamTypes);
1648 return true;
1649 }
1650
1651 unsigned MaskedID =
1652 StringSwitch<unsigned>(Name)
1653 .StartsWith("masked.load", Intrinsic::masked_load)
1654 .StartsWith("masked.gather", Intrinsic::masked_gather)
1655 .StartsWith("masked.store", Intrinsic::masked_store)
1656 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1657 .Default(0);
1658 if (MaskedID && F->arg_size() == 4) {
1659 rename(F);
1660 if (MaskedID == Intrinsic::masked_load ||
1661 MaskedID == Intrinsic::masked_gather) {
1662 NewFn = Intrinsic::getOrInsertDeclaration(
1663 F->getParent(), MaskedID,
1664 {F->getReturnType(), F->getArg(0)->getType()});
1665 return true;
1666 }
1667 NewFn = Intrinsic::getOrInsertDeclaration(
1668 F->getParent(), MaskedID,
1669 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1670 return true;
1671 }
1672 break;
1673 }
1674 case 'n': {
1675 if (Name.consume_front("nvvm.")) {
1676 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1677 if (F->arg_size() == 1) {
1678 Intrinsic::ID IID =
1679 StringSwitch<Intrinsic::ID>(Name)
1680 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1681 .Case("clz.i", Intrinsic::ctlz)
1682 .Case("popc.i", Intrinsic::ctpop)
1683 .Default(Intrinsic::not_intrinsic);
1684 if (IID != Intrinsic::not_intrinsic) {
1685 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1686 {F->getReturnType()});
1687 return true;
1688 }
1689 } else if (F->arg_size() == 2) {
1690 Intrinsic::ID IID =
1691 StringSwitch<Intrinsic::ID>(Name)
1692 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1693 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1694 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1695 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1696 .Default(Intrinsic::not_intrinsic);
1697 if (IID != Intrinsic::not_intrinsic) {
1698 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1699 {F->getReturnType()});
1700 return true;
1701 }
1702 }
1703
1704 // Check for nvvm intrinsics that need a return type adjustment.
1705 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1706 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1707 if (IID != Intrinsic::not_intrinsic) {
1708 NewFn = nullptr;
1709 return true;
1710 }
1711 }
1712
1713 // Upgrade Distributed Shared Memory Intrinsics
1714 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1715 if (IID != Intrinsic::not_intrinsic) {
1716 rename(F);
1717 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1718 return true;
1719 }
1720
1721 // Upgrade TMA copy G2S Intrinsics
1722 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1723 if (IID != Intrinsic::not_intrinsic) {
1724 rename(F);
1725 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1726 return true;
1727 }
1728
1729 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1730 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1731 //
1732 // TODO: We could add lohi.i2d.
1733 bool Expand = false;
1734 if (Name.consume_front("abs."))
1735 // nvvm.abs.{i,ii}
1736 Expand =
1737 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1738 else if (Name.consume_front("fabs."))
1739 // nvvm.fabs.{f,ftz.f,d}
1740 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1741 else if (Name.consume_front("ex2.approx."))
1742 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1743 Expand =
1744 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1745 else if (Name.consume_front("atomic.load."))
1746 // nvvm.atomic.load.add.{f32,f64}.p
1747 // nvvm.atomic.load.{inc,dec}.32.p
1748 Expand = StringSwitch<bool>(Name)
1749 .StartsWith("add.f32.p", true)
1750 .StartsWith("add.f64.p", true)
1751 .StartsWith("inc.32.p", true)
1752 .StartsWith("dec.32.p", true)
1753 .Default(false);
1754 else if (Name.consume_front("atomic."))
1755 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
1756 // nvvm.atomic.cas.gen.i.{cta,sys}
1757 Expand = StringSwitch<bool>(Name)
1758 .StartsWith("add.gen.", true)
1759 .StartsWith("exch.gen.", true)
1760 .StartsWith("max.gen.", true)
1761 .StartsWith("min.gen.", true)
1762 .StartsWith("inc.gen.", true)
1763 .StartsWith("dec.gen.", true)
1764 .StartsWith("and.gen.", true)
1765 .StartsWith("or.gen.", true)
1766 .StartsWith("xor.gen.", true)
1767 .StartsWith("cas.gen.", true)
1768 .Default(false);
1769 else if (Name.consume_front("bitcast."))
1770 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
1771 Expand =
1772 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
1773 else if (Name.consume_front("rotate."))
1774 // nvvm.rotate.{b32,b64,right.b64}
1775 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
1776 else if (Name.consume_front("ptr.gen.to."))
1777 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
1778 Expand = consumeNVVMPtrAddrSpace(Name);
1779 else if (Name.consume_front("ptr."))
1780 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
1781 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
1782 else if (Name.consume_front("ldg.global."))
1783 // nvvm.ldg.global.{i,p,f}
1784 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
1785 Name.starts_with("p."));
1786 else
1787 Expand = StringSwitch<bool>(Name)
1788 .Case("barrier0", true)
1789 .Case("barrier.n", true)
1790 .Case("barrier.sync.cnt", true)
1791 .Case("barrier.sync", true)
1792 .Case("barrier", true)
1793 .Case("bar.sync", true)
1794 .Case("barrier0.popc", true)
1795 .Case("barrier0.and", true)
1796 .Case("barrier0.or", true)
1797 .Case("clz.ll", true)
1798 .Case("popc.ll", true)
1799 .Case("h2f", true)
1800 .Case("swap.lo.hi.b64", true)
1801 .Case("tanh.approx.f32", true)
1802 .Default(false);
1803
1804 if (Expand) {
1805 NewFn = nullptr;
1806 return true;
1807 }
1808 break; // No other 'nvvm.*'.
1809 }
1810 break;
1811 }
1812 case 'o':
1813 if (Name.starts_with("objectsize.")) {
1814 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1815 if (F->arg_size() == 2 || F->arg_size() == 3) {
1816 rename(F);
1817 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1818 Intrinsic::objectsize, Tys);
1819 return true;
1820 }
1821 }
1822 break;
1823
1824 case 'p':
1825 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
1826 rename(F);
1827 NewFn = Intrinsic::getOrInsertDeclaration(
1828 F->getParent(), Intrinsic::ptr_annotation,
1829 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1830 return true;
1831 }
1832 break;
1833
1834 case 'r': {
1835 if (Name.consume_front("riscv.")) {
1836 Intrinsic::ID ID;
1837 ID = StringSwitch<Intrinsic::ID>(Name)
1838 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
1839 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
1840 .Case("aes32esi", Intrinsic::riscv_aes32esi)
1841 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
1842 .Default(Intrinsic::not_intrinsic);
1843 if (ID != Intrinsic::not_intrinsic) {
1844 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1845 rename(F);
1846 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1847 return true;
1848 }
1849 break; // No other applicable upgrades.
1850 }
1851
1852 ID = StringSwitch<Intrinsic::ID>(Name)
1853 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
1854 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
1855 .Default(Intrinsic::not_intrinsic);
1856 if (ID != Intrinsic::not_intrinsic) {
1857 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1858 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1859 rename(F);
1860 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1861 return true;
1862 }
1863 break; // No other applicable upgrades.
1864 }
1865
1866 ID = StringSwitch<Intrinsic::ID>(Name)
1867 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
1868 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
1869 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
1870 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
1871 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
1872 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
1873 .Default(Intrinsic::not_intrinsic);
1874 if (ID != Intrinsic::not_intrinsic) {
1875 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1876 rename(F);
1877 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1878 return true;
1879 }
1880 break; // No other applicable upgrades.
1881 }
1882
1883 // Replace llvm.riscv.clmul with llvm.clmul.
1884 if (Name == "clmul.i32" || Name == "clmul.i64") {
1885 NewFn = Intrinsic::getOrInsertDeclaration(
1886 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
1887 return true;
1888 }
1889
1890 break; // No other 'riscv.*' intrinsics
1891 }
1892 } break;
1893
1894 case 's':
1895 if (Name == "stackprotectorcheck") {
1896 NewFn = nullptr;
1897 return true;
1898 }
1899 break;
1900
1901 case 't':
1902 if (Name == "thread.pointer") {
1903 NewFn = Intrinsic::getOrInsertDeclaration(
1904 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
1905 return true;
1906 }
1907 break;
1908
1909 case 'v': {
1910 if (Name == "var.annotation" && F->arg_size() == 4) {
1911 rename(F);
1912 NewFn = Intrinsic::getOrInsertDeclaration(
1913 F->getParent(), Intrinsic::var_annotation,
1914 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1915 return true;
1916 }
1917 if (Name.consume_front("vector.splice")) {
1918 if (Name.starts_with(".left") || Name.starts_with(".right"))
1919 break;
1920 return true;
1921 }
1922 break;
1923 }
1924
1925 case 'w':
1926 if (Name.consume_front("wasm.")) {
1927 Intrinsic::ID ID =
1928 StringSwitch<Intrinsic::ID>(Name)
1929 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
1930 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
1931 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
1932 .Default(Intrinsic::not_intrinsic);
1933 if (ID != Intrinsic::not_intrinsic) {
1934 rename(F);
1935 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1936 F->getReturnType());
1937 return true;
1938 }
1939
1940 if (Name.consume_front("dot.i8x16.i7x16.")) {
1941 ID = StringSwitch<Intrinsic::ID>(Name)
1942 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1943 .Case("add.signed",
1944 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1945 .Default(Intrinsic::not_intrinsic);
1946 if (ID != Intrinsic::not_intrinsic) {
1947 rename(F);
1948 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1949 return true;
1950 }
1951 break; // No other 'wasm.dot.i8x16.i7x16.*'.
1952 }
1953 break; // No other 'wasm.*'.
1954 }
1955 break;
1956
1957 case 'x':
1958 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
1959 return true;
1960 }
1961
1962 auto *ST = dyn_cast<StructType>(F->getReturnType());
1963 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
1964 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
1965 // Replace return type with literal non-packed struct. Only do this for
1966 // intrinsics declared to return a struct, not for intrinsics with
1967 // overloaded return type, in which case the exact struct type will be
1968 // mangled into the name.
1969 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
1970 FunctionType *FT = F->getFunctionType();
1971 auto *NewST = StructType::get(ST->getContext(), ST->elements());
1972 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1973 std::string Name = F->getName().str();
1974 rename(F);
1975 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1976 Name, F->getParent());
1977
1978 // The new function may also need remangling.
1979 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
1980 NewFn = *Result;
1981 return true;
1982 }
1983 }
1984
1985 // Remangle our intrinsic since we upgrade the mangling
1986 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1987 if (Result != std::nullopt) {
1988 NewFn = *Result;
1989 return true;
1990 }
1991
1992 // This may not belong here. This function is effectively being overloaded
1993 // to both detect an intrinsic which needs upgrading, and to provide the
1994 // upgraded form of the intrinsic. We should perhaps have two separate
1995 // functions for this.
1996 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
1997 return true;
1998
1999 return false;
2000}
2001
2002bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2003 bool CanUpgradeDebugIntrinsicsToRecords) {
2004 NewFn = nullptr;
2005 bool Upgraded =
2006 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2007
2008 // Upgrade intrinsic attributes. This does not change the function.
2009 if (NewFn)
2010 F = NewFn;
2011 if (Intrinsic::ID id = F->getIntrinsicID()) {
2012 // Only do this if the intrinsic signature is valid.
2013 SmallVector<Type *> OverloadTys;
2014 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2015 F->setAttributes(
2016 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2017 }
2018 return Upgraded;
2019}
2020
2021GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2022 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2023 GV->getName() == "llvm.global_dtors")) ||
2024 !GV->hasInitializer())
2025 return nullptr;
2026 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2027 if (!ATy)
2028 return nullptr;
2029 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2030 if (!STy || STy->getNumElements() != 2)
2031 return nullptr;
2032
2033 LLVMContext &C = GV->getContext();
2034 IRBuilder<> IRB(C);
2035 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2036 IRB.getPtrTy());
2037 Constant *Init = GV->getInitializer();
2038 unsigned N = Init->getNumOperands();
2039 std::vector<Constant *> NewCtors(N);
2040 for (unsigned i = 0; i != N; ++i) {
2041 auto Ctor = cast<Constant>(Init->getOperand(i));
2042 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2043 Ctor->getAggregateElement(1),
2044 ConstantPointerNull::get(IRB.getPtrTy()));
2045 }
2046 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2047
2048 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2049 NewInit, GV->getName());
2050}
2051
2052// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2053// to byte shuffles.
2054static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2055 unsigned Shift) {
2056 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2057 unsigned NumElts = ResultTy->getNumElements() * 8;
2058
2059 // Bitcast from a 64-bit element type to a byte element type.
2060 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2061 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2062
2063 // We'll be shuffling in zeroes.
2064 Value *Res = Constant::getNullValue(VecTy);
2065
2066 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2067 // we'll just return the zero vector.
2068 if (Shift < 16) {
2069 int Idxs[64];
2070 // 256/512-bit version is split into 2/4 16-byte lanes.
2071 for (unsigned l = 0; l != NumElts; l += 16)
2072 for (unsigned i = 0; i != 16; ++i) {
2073 unsigned Idx = NumElts + i - Shift;
2074 if (Idx < NumElts)
2075 Idx -= NumElts - 16; // end of lane, switch operand.
2076 Idxs[l + i] = Idx + l;
2077 }
2078
2079 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2080 }
2081
2082 // Bitcast back to a 64-bit element type.
2083 return Builder.CreateBitCast(Res, ResultTy, "cast");
2084}
2085
2086// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2087// to byte shuffles.
2088static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2089 unsigned Shift) {
2090 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2091 unsigned NumElts = ResultTy->getNumElements() * 8;
2092
2093 // Bitcast from a 64-bit element type to a byte element type.
2094 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2095 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2096
2097 // We'll be shuffling in zeroes.
2098 Value *Res = Constant::getNullValue(VecTy);
2099
2100 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2101 // we'll just return the zero vector.
2102 if (Shift < 16) {
2103 int Idxs[64];
2104 // 256/512-bit version is split into 2/4 16-byte lanes.
2105 for (unsigned l = 0; l != NumElts; l += 16)
2106 for (unsigned i = 0; i != 16; ++i) {
2107 unsigned Idx = i + Shift;
2108 if (Idx >= 16)
2109 Idx += NumElts - 16; // end of lane, switch operand.
2110 Idxs[l + i] = Idx + l;
2111 }
2112
2113 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2114 }
2115
2116 // Bitcast back to a 64-bit element type.
2117 return Builder.CreateBitCast(Res, ResultTy, "cast");
2118}
2119
2120static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2121 unsigned NumElts) {
2122 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2123 llvm::VectorType *MaskTy = FixedVectorType::get(
2124 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2125 Mask = Builder.CreateBitCast(Mask, MaskTy);
2126
2127 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2128 // i8 and we need to extract down to the right number of elements.
2129 if (NumElts <= 4) {
2130 int Indices[4];
2131 for (unsigned i = 0; i != NumElts; ++i)
2132 Indices[i] = i;
2133 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2134 "extract");
2135 }
2136
2137 return Mask;
2138}
2139
2140static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2141 Value *Op1) {
2142 // If the mask is all ones just emit the first operation.
2143 if (const auto *C = dyn_cast<Constant>(Mask))
2144 if (C->isAllOnesValue())
2145 return Op0;
2146
2147 Mask = getX86MaskVec(Builder, Mask,
2148 cast<FixedVectorType>(Op0->getType())->getNumElements());
2149 return Builder.CreateSelect(Mask, Op0, Op1);
2150}
2151
2152static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2153 Value *Op1) {
2154 // If the mask is all ones just emit the first operation.
2155 if (const auto *C = dyn_cast<Constant>(Mask))
2156 if (C->isAllOnesValue())
2157 return Op0;
2158
2159 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2160 Mask->getType()->getIntegerBitWidth());
2161 Mask = Builder.CreateBitCast(Mask, MaskTy);
2162 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2163 return Builder.CreateSelect(Mask, Op0, Op1);
2164}
2165
2166// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2167// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2168// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2169static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2170 Value *Op1, Value *Shift,
2171 Value *Passthru, Value *Mask,
2172 bool IsVALIGN) {
2173 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2174
2175 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2176 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2177 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2178 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2179
2180 // Mask the immediate for VALIGN.
2181 if (IsVALIGN)
2182 ShiftVal &= (NumElts - 1);
2183
2184 // If palignr is shifting the pair of vectors more than the size of two
2185 // lanes, emit zero.
2186 if (ShiftVal >= 32)
2187 return llvm::Constant::getNullValue(Op0->getType());
2188
2189 // If palignr is shifting the pair of input vectors more than one lane,
2190 // but less than two lanes, convert to shifting in zeroes.
2191 if (ShiftVal > 16) {
2192 ShiftVal -= 16;
2193 Op1 = Op0;
2194 Op0 = llvm::Constant::getNullValue(Op0->getType());
2195 }
2196
2197 int Indices[64];
2198 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2199 for (unsigned l = 0; l < NumElts; l += 16) {
2200 for (unsigned i = 0; i != 16; ++i) {
2201 unsigned Idx = ShiftVal + i;
2202 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2203 Idx += NumElts - 16; // End of lane, switch operand.
2204 Indices[l + i] = Idx + l;
2205 }
2206 }
2207
2208 Value *Align = Builder.CreateShuffleVector(
2209 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2210
2211 return emitX86Select(Builder, Mask, Align, Passthru);
2212}
2213
2214static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2215 bool ZeroMask, bool IndexForm) {
2216 Type *Ty = CI.getType();
2217 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2218 unsigned EltWidth = Ty->getScalarSizeInBits();
2219 bool IsFloat = Ty->isFPOrFPVectorTy();
2220 Intrinsic::ID IID;
2221 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2222 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2223 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2224 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2225 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2226 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2227 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2228 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2229 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2230 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2231 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2232 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2233 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2234 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2235 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2236 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2237 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2238 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2239 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2240 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2241 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2242 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2243 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2244 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2245 else if (VecWidth == 128 && EltWidth == 16)
2246 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2247 else if (VecWidth == 256 && EltWidth == 16)
2248 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2249 else if (VecWidth == 512 && EltWidth == 16)
2250 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2251 else if (VecWidth == 128 && EltWidth == 8)
2252 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2253 else if (VecWidth == 256 && EltWidth == 8)
2254 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2255 else if (VecWidth == 512 && EltWidth == 8)
2256 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2257 else
2258 llvm_unreachable("Unexpected intrinsic");
2259
2260 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2261 CI.getArgOperand(2) };
2262
2263 // If this isn't index form we need to swap operand 0 and 1.
2264 if (!IndexForm)
2265 std::swap(Args[0], Args[1]);
2266
2267 Value *V = Builder.CreateIntrinsic(IID, Args);
2268 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2269 : Builder.CreateBitCast(CI.getArgOperand(1),
2270 Ty);
2271 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2272}
2273
2274static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2275 Intrinsic::ID IID) {
2276 Type *Ty = CI.getType();
2277 Value *Op0 = CI.getOperand(0);
2278 Value *Op1 = CI.getOperand(1);
2279 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2280
2281 if (CI.arg_size() == 4) { // For masked intrinsics.
2282 Value *VecSrc = CI.getOperand(2);
2283 Value *Mask = CI.getOperand(3);
2284 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2285 }
2286 return Res;
2287}
2288
2289static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2290 bool IsRotateRight) {
2291 Type *Ty = CI.getType();
2292 Value *Src = CI.getArgOperand(0);
2293 Value *Amt = CI.getArgOperand(1);
2294
2295 // Amount may be scalar immediate, in which case create a splat vector.
2296 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2297 // we only care about the lowest log2 bits anyway.
2298 if (Amt->getType() != Ty) {
2299 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2300 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2301 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2302 }
2303
2304 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2305 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2306
2307 if (CI.arg_size() == 4) { // For masked intrinsics.
2308 Value *VecSrc = CI.getOperand(2);
2309 Value *Mask = CI.getOperand(3);
2310 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2311 }
2312 return Res;
2313}
2314
2315static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2316 bool IsSigned) {
2317 Type *Ty = CI.getType();
2318 Value *LHS = CI.getArgOperand(0);
2319 Value *RHS = CI.getArgOperand(1);
2320
2321 CmpInst::Predicate Pred;
2322 switch (Imm) {
2323 case 0x0:
2324 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2325 break;
2326 case 0x1:
2327 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2328 break;
2329 case 0x2:
2330 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2331 break;
2332 case 0x3:
2333 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2334 break;
2335 case 0x4:
2336 Pred = ICmpInst::ICMP_EQ;
2337 break;
2338 case 0x5:
2339 Pred = ICmpInst::ICMP_NE;
2340 break;
2341 case 0x6:
2342 return Constant::getNullValue(Ty); // FALSE
2343 case 0x7:
2344 return Constant::getAllOnesValue(Ty); // TRUE
2345 default:
2346 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2347 }
2348
2349 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2350 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2351 return Ext;
2352}
2353
2354static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2355 bool IsShiftRight, bool ZeroMask) {
2356 Type *Ty = CI.getType();
2357 Value *Op0 = CI.getArgOperand(0);
2358 Value *Op1 = CI.getArgOperand(1);
2359 Value *Amt = CI.getArgOperand(2);
2360
2361 if (IsShiftRight)
2362 std::swap(Op0, Op1);
2363
2364 // Amount may be scalar immediate, in which case create a splat vector.
2365 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2366 // we only care about the lowest log2 bits anyway.
2367 if (Amt->getType() != Ty) {
2368 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2369 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2370 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2371 }
2372
2373 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2374 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2375
2376 unsigned NumArgs = CI.arg_size();
2377 if (NumArgs >= 4) { // For masked intrinsics.
2378 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2379 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2380 CI.getArgOperand(0);
2381 Value *Mask = CI.getOperand(NumArgs - 1);
2382 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2383 }
2384 return Res;
2385}
2386
2387static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2388 Value *Mask, bool Aligned) {
2389 const Align Alignment =
2390 Aligned
2391 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2392 : Align(1);
2393
2394 // If the mask is all ones just emit a regular store.
2395 if (const auto *C = dyn_cast<Constant>(Mask))
2396 if (C->isAllOnesValue())
2397 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2398
2399 // Convert the mask from an integer type to a vector of i1.
2400 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2401 Mask = getX86MaskVec(Builder, Mask, NumElts);
2402 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2403}
2404
2405static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2406 Value *Passthru, Value *Mask, bool Aligned) {
2407 Type *ValTy = Passthru->getType();
2408 const Align Alignment =
2409 Aligned
2410 ? Align(
2411 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2412 8)
2413 : Align(1);
2414
2415 // If the mask is all ones just emit a regular store.
2416 if (const auto *C = dyn_cast<Constant>(Mask))
2417 if (C->isAllOnesValue())
2418 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2419
2420 // Convert the mask from an integer type to a vector of i1.
2421 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2422 Mask = getX86MaskVec(Builder, Mask, NumElts);
2423 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2424}
2425
2426static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2427 Type *Ty = CI.getType();
2428 Value *Op0 = CI.getArgOperand(0);
2429 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2430 {Op0, Builder.getInt1(false)});
2431 if (CI.arg_size() == 3)
2432 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2433 return Res;
2434}
2435
2436static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2437 Type *Ty = CI.getType();
2438
2439 // Arguments have a vXi32 type so cast to vXi64.
2440 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2441 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2442
2443 if (IsSigned) {
2444 // Shift left then arithmetic shift right.
2445 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2446 LHS = Builder.CreateShl(LHS, ShiftAmt);
2447 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2448 RHS = Builder.CreateShl(RHS, ShiftAmt);
2449 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2450 } else {
2451 // Clear the upper bits.
2452 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2453 LHS = Builder.CreateAnd(LHS, Mask);
2454 RHS = Builder.CreateAnd(RHS, Mask);
2455 }
2456
2457 Value *Res = Builder.CreateMul(LHS, RHS);
2458
2459 if (CI.arg_size() == 4)
2460 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2461
2462 return Res;
2463}
2464
2465// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2466static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2467 Value *Mask) {
2468 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2469 if (Mask) {
2470 const auto *C = dyn_cast<Constant>(Mask);
2471 if (!C || !C->isAllOnesValue())
2472 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2473 }
2474
2475 if (NumElts < 8) {
2476 int Indices[8];
2477 for (unsigned i = 0; i != NumElts; ++i)
2478 Indices[i] = i;
2479 for (unsigned i = NumElts; i != 8; ++i)
2480 Indices[i] = NumElts + i % NumElts;
2481 Vec = Builder.CreateShuffleVector(Vec,
2482 Constant::getNullValue(Vec->getType()),
2483 Indices);
2484 }
2485 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2486}
2487
2488static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2489 unsigned CC, bool Signed) {
2490 Value *Op0 = CI.getArgOperand(0);
2491 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2492
2493 Value *Cmp;
2494 if (CC == 3) {
2495 Cmp = Constant::getNullValue(
2496 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2497 } else if (CC == 7) {
2498 Cmp = Constant::getAllOnesValue(
2499 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2500 } else {
2501 ICmpInst::Predicate Pred;
2502 switch (CC) {
2503 default: llvm_unreachable("Unknown condition code");
2504 case 0: Pred = ICmpInst::ICMP_EQ; break;
2505 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2506 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2507 case 4: Pred = ICmpInst::ICMP_NE; break;
2508 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2509 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2510 }
2511 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2512 }
2513
2514 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2515
2516 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2517}
2518
2519// Replace a masked intrinsic with an older unmasked intrinsic.
2520static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2521 Intrinsic::ID IID) {
2522 Value *Rep =
2523 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2524 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2525}
2526
2527static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2528 Value* A = CI.getArgOperand(0);
2529 Value* B = CI.getArgOperand(1);
2530 Value* Src = CI.getArgOperand(2);
2531 Value* Mask = CI.getArgOperand(3);
2532
2533 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2534 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2535 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2536 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2537 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2538 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2539}
2540
2541static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2542 Value* Op = CI.getArgOperand(0);
2543 Type* ReturnOp = CI.getType();
2544 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2545 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2546 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2547}
2548
2549// Replace intrinsic with unmasked version and a select.
2550static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2551 CallBase &CI, Value *&Rep) {
2552 Name = Name.substr(12); // Remove avx512.mask.
2553
2554 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2555 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2556 Intrinsic::ID IID;
2557 if (Name.starts_with("max.p")) {
2558 if (VecWidth == 128 && EltWidth == 32)
2559 IID = Intrinsic::x86_sse_max_ps;
2560 else if (VecWidth == 128 && EltWidth == 64)
2561 IID = Intrinsic::x86_sse2_max_pd;
2562 else if (VecWidth == 256 && EltWidth == 32)
2563 IID = Intrinsic::x86_avx_max_ps_256;
2564 else if (VecWidth == 256 && EltWidth == 64)
2565 IID = Intrinsic::x86_avx_max_pd_256;
2566 else
2567 llvm_unreachable("Unexpected intrinsic");
2568 } else if (Name.starts_with("min.p")) {
2569 if (VecWidth == 128 && EltWidth == 32)
2570 IID = Intrinsic::x86_sse_min_ps;
2571 else if (VecWidth == 128 && EltWidth == 64)
2572 IID = Intrinsic::x86_sse2_min_pd;
2573 else if (VecWidth == 256 && EltWidth == 32)
2574 IID = Intrinsic::x86_avx_min_ps_256;
2575 else if (VecWidth == 256 && EltWidth == 64)
2576 IID = Intrinsic::x86_avx_min_pd_256;
2577 else
2578 llvm_unreachable("Unexpected intrinsic");
2579 } else if (Name.starts_with("pshuf.b.")) {
2580 if (VecWidth == 128)
2581 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2582 else if (VecWidth == 256)
2583 IID = Intrinsic::x86_avx2_pshuf_b;
2584 else if (VecWidth == 512)
2585 IID = Intrinsic::x86_avx512_pshuf_b_512;
2586 else
2587 llvm_unreachable("Unexpected intrinsic");
2588 } else if (Name.starts_with("pmul.hr.sw.")) {
2589 if (VecWidth == 128)
2590 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2591 else if (VecWidth == 256)
2592 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2593 else if (VecWidth == 512)
2594 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2595 else
2596 llvm_unreachable("Unexpected intrinsic");
2597 } else if (Name.starts_with("pmulh.w.")) {
2598 if (VecWidth == 128)
2599 IID = Intrinsic::x86_sse2_pmulh_w;
2600 else if (VecWidth == 256)
2601 IID = Intrinsic::x86_avx2_pmulh_w;
2602 else if (VecWidth == 512)
2603 IID = Intrinsic::x86_avx512_pmulh_w_512;
2604 else
2605 llvm_unreachable("Unexpected intrinsic");
2606 } else if (Name.starts_with("pmulhu.w.")) {
2607 if (VecWidth == 128)
2608 IID = Intrinsic::x86_sse2_pmulhu_w;
2609 else if (VecWidth == 256)
2610 IID = Intrinsic::x86_avx2_pmulhu_w;
2611 else if (VecWidth == 512)
2612 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2613 else
2614 llvm_unreachable("Unexpected intrinsic");
2615 } else if (Name.starts_with("pmaddw.d.")) {
2616 if (VecWidth == 128)
2617 IID = Intrinsic::x86_sse2_pmadd_wd;
2618 else if (VecWidth == 256)
2619 IID = Intrinsic::x86_avx2_pmadd_wd;
2620 else if (VecWidth == 512)
2621 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2622 else
2623 llvm_unreachable("Unexpected intrinsic");
2624 } else if (Name.starts_with("pmaddubs.w.")) {
2625 if (VecWidth == 128)
2626 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2627 else if (VecWidth == 256)
2628 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2629 else if (VecWidth == 512)
2630 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2631 else
2632 llvm_unreachable("Unexpected intrinsic");
2633 } else if (Name.starts_with("packsswb.")) {
2634 if (VecWidth == 128)
2635 IID = Intrinsic::x86_sse2_packsswb_128;
2636 else if (VecWidth == 256)
2637 IID = Intrinsic::x86_avx2_packsswb;
2638 else if (VecWidth == 512)
2639 IID = Intrinsic::x86_avx512_packsswb_512;
2640 else
2641 llvm_unreachable("Unexpected intrinsic");
2642 } else if (Name.starts_with("packssdw.")) {
2643 if (VecWidth == 128)
2644 IID = Intrinsic::x86_sse2_packssdw_128;
2645 else if (VecWidth == 256)
2646 IID = Intrinsic::x86_avx2_packssdw;
2647 else if (VecWidth == 512)
2648 IID = Intrinsic::x86_avx512_packssdw_512;
2649 else
2650 llvm_unreachable("Unexpected intrinsic");
2651 } else if (Name.starts_with("packuswb.")) {
2652 if (VecWidth == 128)
2653 IID = Intrinsic::x86_sse2_packuswb_128;
2654 else if (VecWidth == 256)
2655 IID = Intrinsic::x86_avx2_packuswb;
2656 else if (VecWidth == 512)
2657 IID = Intrinsic::x86_avx512_packuswb_512;
2658 else
2659 llvm_unreachable("Unexpected intrinsic");
2660 } else if (Name.starts_with("packusdw.")) {
2661 if (VecWidth == 128)
2662 IID = Intrinsic::x86_sse41_packusdw;
2663 else if (VecWidth == 256)
2664 IID = Intrinsic::x86_avx2_packusdw;
2665 else if (VecWidth == 512)
2666 IID = Intrinsic::x86_avx512_packusdw_512;
2667 else
2668 llvm_unreachable("Unexpected intrinsic");
2669 } else if (Name.starts_with("vpermilvar.")) {
2670 if (VecWidth == 128 && EltWidth == 32)
2671 IID = Intrinsic::x86_avx_vpermilvar_ps;
2672 else if (VecWidth == 128 && EltWidth == 64)
2673 IID = Intrinsic::x86_avx_vpermilvar_pd;
2674 else if (VecWidth == 256 && EltWidth == 32)
2675 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2676 else if (VecWidth == 256 && EltWidth == 64)
2677 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2678 else if (VecWidth == 512 && EltWidth == 32)
2679 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2680 else if (VecWidth == 512 && EltWidth == 64)
2681 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2682 else
2683 llvm_unreachable("Unexpected intrinsic");
2684 } else if (Name == "cvtpd2dq.256") {
2685 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2686 } else if (Name == "cvtpd2ps.256") {
2687 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2688 } else if (Name == "cvttpd2dq.256") {
2689 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2690 } else if (Name == "cvttps2dq.128") {
2691 IID = Intrinsic::x86_sse2_cvttps2dq;
2692 } else if (Name == "cvttps2dq.256") {
2693 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2694 } else if (Name.starts_with("permvar.")) {
2695 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2696 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2697 IID = Intrinsic::x86_avx2_permps;
2698 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2699 IID = Intrinsic::x86_avx2_permd;
2700 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2701 IID = Intrinsic::x86_avx512_permvar_df_256;
2702 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2703 IID = Intrinsic::x86_avx512_permvar_di_256;
2704 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2705 IID = Intrinsic::x86_avx512_permvar_sf_512;
2706 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2707 IID = Intrinsic::x86_avx512_permvar_si_512;
2708 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2709 IID = Intrinsic::x86_avx512_permvar_df_512;
2710 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2711 IID = Intrinsic::x86_avx512_permvar_di_512;
2712 else if (VecWidth == 128 && EltWidth == 16)
2713 IID = Intrinsic::x86_avx512_permvar_hi_128;
2714 else if (VecWidth == 256 && EltWidth == 16)
2715 IID = Intrinsic::x86_avx512_permvar_hi_256;
2716 else if (VecWidth == 512 && EltWidth == 16)
2717 IID = Intrinsic::x86_avx512_permvar_hi_512;
2718 else if (VecWidth == 128 && EltWidth == 8)
2719 IID = Intrinsic::x86_avx512_permvar_qi_128;
2720 else if (VecWidth == 256 && EltWidth == 8)
2721 IID = Intrinsic::x86_avx512_permvar_qi_256;
2722 else if (VecWidth == 512 && EltWidth == 8)
2723 IID = Intrinsic::x86_avx512_permvar_qi_512;
2724 else
2725 llvm_unreachable("Unexpected intrinsic");
2726 } else if (Name.starts_with("dbpsadbw.")) {
2727 if (VecWidth == 128)
2728 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2729 else if (VecWidth == 256)
2730 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2731 else if (VecWidth == 512)
2732 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2733 else
2734 llvm_unreachable("Unexpected intrinsic");
2735 } else if (Name.starts_with("pmultishift.qb.")) {
2736 if (VecWidth == 128)
2737 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2738 else if (VecWidth == 256)
2739 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2740 else if (VecWidth == 512)
2741 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2742 else
2743 llvm_unreachable("Unexpected intrinsic");
2744 } else if (Name.starts_with("conflict.")) {
2745 if (Name[9] == 'd' && VecWidth == 128)
2746 IID = Intrinsic::x86_avx512_conflict_d_128;
2747 else if (Name[9] == 'd' && VecWidth == 256)
2748 IID = Intrinsic::x86_avx512_conflict_d_256;
2749 else if (Name[9] == 'd' && VecWidth == 512)
2750 IID = Intrinsic::x86_avx512_conflict_d_512;
2751 else if (Name[9] == 'q' && VecWidth == 128)
2752 IID = Intrinsic::x86_avx512_conflict_q_128;
2753 else if (Name[9] == 'q' && VecWidth == 256)
2754 IID = Intrinsic::x86_avx512_conflict_q_256;
2755 else if (Name[9] == 'q' && VecWidth == 512)
2756 IID = Intrinsic::x86_avx512_conflict_q_512;
2757 else
2758 llvm_unreachable("Unexpected intrinsic");
2759 } else if (Name.starts_with("pavg.")) {
2760 if (Name[5] == 'b' && VecWidth == 128)
2761 IID = Intrinsic::x86_sse2_pavg_b;
2762 else if (Name[5] == 'b' && VecWidth == 256)
2763 IID = Intrinsic::x86_avx2_pavg_b;
2764 else if (Name[5] == 'b' && VecWidth == 512)
2765 IID = Intrinsic::x86_avx512_pavg_b_512;
2766 else if (Name[5] == 'w' && VecWidth == 128)
2767 IID = Intrinsic::x86_sse2_pavg_w;
2768 else if (Name[5] == 'w' && VecWidth == 256)
2769 IID = Intrinsic::x86_avx2_pavg_w;
2770 else if (Name[5] == 'w' && VecWidth == 512)
2771 IID = Intrinsic::x86_avx512_pavg_w_512;
2772 else
2773 llvm_unreachable("Unexpected intrinsic");
2774 } else
2775 return false;
2776
2777 SmallVector<Value *, 4> Args(CI.args());
2778 Args.pop_back();
2779 Args.pop_back();
2780 Rep = Builder.CreateIntrinsic(IID, Args);
2781 unsigned NumArgs = CI.arg_size();
2782 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2783 CI.getArgOperand(NumArgs - 2));
2784 return true;
2785}
2786
2787/// Upgrade comment in call to inline asm that represents an objc retain release
2788/// marker.
2789void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2790 size_t Pos;
2791 if (AsmStr->find("mov\tfp") == 0 &&
2792 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
2793 (Pos = AsmStr->find("# marker")) != std::string::npos) {
2794 AsmStr->replace(Pos, 1, ";");
2795 }
2796}
2797
2798static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
2799 Function *F, IRBuilder<> &Builder) {
2800 Value *Rep = nullptr;
2801
2802 if (Name == "abs.i" || Name == "abs.ll") {
2803 Value *Arg = CI->getArgOperand(0);
2804 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
2805 {Arg, Builder.getTrue()},
2806 /*FMFSource=*/nullptr, "abs");
2807 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
2808 Type *Ty = (Name == "abs.bf16")
2809 ? Builder.getBFloatTy()
2810 : FixedVectorType::get(Builder.getBFloatTy(), 2);
2811 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
2812 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
2813 Rep = Builder.CreateBitCast(Abs, CI->getType());
2814 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
2815 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
2816 : Intrinsic::nvvm_fabs;
2817 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2818 } else if (Name.consume_front("ex2.approx.")) {
2819 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2820 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
2821 : Intrinsic::nvvm_ex2_approx;
2822 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2823 } else if (Name.starts_with("atomic.load.add.f32.p") ||
2824 Name.starts_with("atomic.load.add.f64.p")) {
2825 Value *Ptr = CI->getArgOperand(0);
2826 Value *Val = CI->getArgOperand(1);
2827 Rep = Builder.CreateAtomicRMW(
2828 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2829 CI->getContext().getOrInsertSyncScopeID("device"));
2830 // The default scope for atomic.load.* intrinsics is device
2831 // (= gpu scope in ptx), but the default LLVM atomic scope is
2832 // "system"
2833 } else if (Name.starts_with("atomic.load.inc.32.p") ||
2834 Name.starts_with("atomic.load.dec.32.p")) {
2835 Value *Ptr = CI->getArgOperand(0);
2836 Value *Val = CI->getArgOperand(1);
2837 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
2838 : AtomicRMWInst::UDecWrap;
2839 Rep = Builder.CreateAtomicRMW(
2840 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2841 CI->getContext().getOrInsertSyncScopeID("device"));
2842 // See comment above.
2843 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
2844 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
2845 StringRef Op = Name.substr(StringRef("atomic.").size());
2846 Value *Ptr = CI->getArgOperand(0);
2847 Value *Val = CI->getArgOperand(1);
2848 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
2849 Op.contains(".cta.") ? "block" : "");
2850 if (Op.starts_with("cas.")) {
2851 Value *New = CI->getArgOperand(2);
2852 Value *Pair = Builder.CreateAtomicCmpXchg(
2853 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
2854 AtomicOrdering::Monotonic, SSID);
2855 Rep = Builder.CreateExtractValue(Pair, 0);
2856 } else {
2857 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
2858 // because we were actually missing those intrinsics!
2859 AtomicRMWInst::BinOp BinOp =
2860 StringSwitch<AtomicRMWInst::BinOp>(Op)
2861 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
2862 .StartsWith("add.gen.i", AtomicRMWInst::Add)
2863 .StartsWith("exch.", AtomicRMWInst::Xchg)
2864 .StartsWith("max.", AtomicRMWInst::Max)
2865 .StartsWith("min.", AtomicRMWInst::Min)
2866 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
2867 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
2868 .StartsWith("and.", AtomicRMWInst::And)
2869 .StartsWith("or.", AtomicRMWInst::Or)
2870 .StartsWith("xor.", AtomicRMWInst::Xor)
2871 .Default(AtomicRMWInst::BAD_BINOP);
2872 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
2873 "unexpected nvvm scoped atomic intrinsic");
2874 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
2875 AtomicOrdering::Monotonic, SSID);
2876 }
2877 } else if (Name == "clz.ll") {
2878 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
2879 Value *Arg = CI->getArgOperand(0);
2880 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
2881 {Arg, Builder.getFalse()},
2882 /*FMFSource=*/nullptr, "ctlz");
2883 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
2884 } else if (Name == "popc.ll") {
2885 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
2886 // i64.
2887 Value *Arg = CI->getArgOperand(0);
2888 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
2889 Arg, /*FMFSource=*/nullptr, "ctpop");
2890 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
2891 } else if (Name == "h2f") {
2892 Value *Cast =
2893 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
2894 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
2895 } else if (Name.consume_front("bitcast.") &&
2896 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
2897 Name == "d2ll")) {
2898 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
2899 } else if (Name == "rotate.b32") {
2900 Value *Arg = CI->getOperand(0);
2901 Value *ShiftAmt = CI->getOperand(1);
2902 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
2903 {Arg, Arg, ShiftAmt});
2904 } else if (Name == "rotate.b64") {
2905 Type *Int64Ty = Builder.getInt64Ty();
2906 Value *Arg = CI->getOperand(0);
2907 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2908 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2909 {Arg, Arg, ZExtShiftAmt});
2910 } else if (Name == "rotate.right.b64") {
2911 Type *Int64Ty = Builder.getInt64Ty();
2912 Value *Arg = CI->getOperand(0);
2913 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2914 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
2915 {Arg, Arg, ZExtShiftAmt});
2916 } else if (Name == "swap.lo.hi.b64") {
2917 Type *Int64Ty = Builder.getInt64Ty();
2918 Value *Arg = CI->getOperand(0);
2919 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2920 {Arg, Arg, Builder.getInt64(32)});
2921 } else if ((Name.consume_front("ptr.gen.to.") &&
2922 consumeNVVMPtrAddrSpace(Name)) ||
2923 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
2924 Name.starts_with(".to.gen"))) {
2925 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
2926 } else if (Name.consume_front("ldg.global")) {
2927 Value *Ptr = CI->getArgOperand(0);
2928 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
2929 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
2930 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
2931 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
2932 MDNode *MD = MDNode::get(Builder.getContext(), {});
2933 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
2934 return LD;
2935 } else if (Name == "tanh.approx.f32") {
2936 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
2937 FastMathFlags FMF;
2938 FMF.setApproxFunc();
2939 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
2940 FMF);
2941 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
2942 Value *Arg =
2943 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
2944 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
2945 {}, {Arg});
2946 } else if (Name == "barrier") {
2947 Rep = Builder.CreateIntrinsic(
2948 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
2949 {CI->getArgOperand(0), CI->getArgOperand(1)});
2950 } else if (Name == "barrier.sync") {
2951 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
2952 {CI->getArgOperand(0)});
2953 } else if (Name == "barrier.sync.cnt") {
2954 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
2955 {CI->getArgOperand(0), CI->getArgOperand(1)});
2956 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
2957 Name == "barrier0.or") {
2958 Value *C = CI->getArgOperand(0);
2959 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
2960
2961 Intrinsic::ID IID =
2962 StringSwitch<Intrinsic::ID>(Name)
2963 .Case("barrier0.popc",
2964 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
2965 .Case("barrier0.and",
2966 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
2967 .Case("barrier0.or",
2968 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
2969 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
2970 Rep = Builder.CreateZExt(Bar, CI->getType());
2971 } else {
2972 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2973 if (IID != Intrinsic::not_intrinsic &&
2974 !F->getReturnType()->getScalarType()->isBFloatTy()) {
2975 rename(F);
2976 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2977 SmallVector<Value *, 2> Args;
2978 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
2979 Value *Arg = CI->getArgOperand(I);
2980 Type *OldType = Arg->getType();
2981 Type *NewType = NewFn->getArg(I)->getType();
2982 Args.push_back(
2983 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
2984 ? Builder.CreateBitCast(Arg, NewType)
2985 : Arg);
2986 }
2987 Rep = Builder.CreateCall(NewFn, Args);
2988 if (F->getReturnType()->isIntegerTy())
2989 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
2990 }
2991 }
2992
2993 return Rep;
2994}
2995
2996static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2997 IRBuilder<> &Builder) {
2998 LLVMContext &C = F->getContext();
2999 Value *Rep = nullptr;
3000
3001 if (Name.starts_with("sse4a.movnt.")) {
3002 SmallVector<Metadata *, 1> Elts;
3003 Elts.push_back(
3004 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3005 MDNode *Node = MDNode::get(C, Elts);
3006
3007 Value *Arg0 = CI->getArgOperand(0);
3008 Value *Arg1 = CI->getArgOperand(1);
3009
3010 // Nontemporal (unaligned) store of the 0'th element of the float/double
3011 // vector.
3012 Value *Extract =
3013 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3014
3015 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3016 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3017 } else if (Name.starts_with("avx.movnt.") ||
3018 Name.starts_with("avx512.storent.")) {
3019 SmallVector<Metadata *, 1> Elts;
3020 Elts.push_back(
3021 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3022 MDNode *Node = MDNode::get(C, Elts);
3023
3024 Value *Arg0 = CI->getArgOperand(0);
3025 Value *Arg1 = CI->getArgOperand(1);
3026
3027 StoreInst *SI = Builder.CreateAlignedStore(
3028 Arg1, Arg0,
3029 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3030 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3031 } else if (Name == "sse2.storel.dq") {
3032 Value *Arg0 = CI->getArgOperand(0);
3033 Value *Arg1 = CI->getArgOperand(1);
3034
3035 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3036 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3037 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3038 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3039 } else if (Name.starts_with("sse.storeu.") ||
3040 Name.starts_with("sse2.storeu.") ||
3041 Name.starts_with("avx.storeu.")) {
3042 Value *Arg0 = CI->getArgOperand(0);
3043 Value *Arg1 = CI->getArgOperand(1);
3044 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3045 } else if (Name == "avx512.mask.store.ss") {
3046 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3047 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3048 Mask, false);
3049 } else if (Name.starts_with("avx512.mask.store")) {
3050 // "avx512.mask.storeu." or "avx512.mask.store."
3051 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3052 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3053 CI->getArgOperand(2), Aligned);
3054 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3055 // Upgrade packed integer vector compare intrinsics to compare instructions.
3056 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3057 bool CmpEq = Name[9] == 'e';
3058 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3059 CI->getArgOperand(0), CI->getArgOperand(1));
3060 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3061 } else if (Name.starts_with("avx512.broadcastm")) {
3062 Type *ExtTy = Type::getInt32Ty(C);
3063 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3064 ExtTy = Type::getInt64Ty(C);
3065 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3066 ExtTy->getPrimitiveSizeInBits();
3067 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3068 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3069 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3070 Value *Vec = CI->getArgOperand(0);
3071 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3072 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3073 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3074 } else if (Name.starts_with("avx.sqrt.p") ||
3075 Name.starts_with("sse2.sqrt.p") ||
3076 Name.starts_with("sse.sqrt.p")) {
3077 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3078 {CI->getArgOperand(0)});
3079 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3080 if (CI->arg_size() == 4 &&
3081 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3082 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3083 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3084 : Intrinsic::x86_avx512_sqrt_pd_512;
3085
3086 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3087 Rep = Builder.CreateIntrinsic(IID, Args);
3088 } else {
3089 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3090 {CI->getArgOperand(0)});
3091 }
3092 Rep =
3093 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3094 } else if (Name.starts_with("avx512.ptestm") ||
3095 Name.starts_with("avx512.ptestnm")) {
3096 Value *Op0 = CI->getArgOperand(0);
3097 Value *Op1 = CI->getArgOperand(1);
3098 Value *Mask = CI->getArgOperand(2);
3099 Rep = Builder.CreateAnd(Op0, Op1);
3100 llvm::Type *Ty = Op0->getType();
3101 Value *Zero = llvm::Constant::getNullValue(Ty);
3102 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3103 ? ICmpInst::ICMP_NE
3104 : ICmpInst::ICMP_EQ;
3105 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3106 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3107 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3108 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3109 ->getNumElements();
3110 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3111 Rep =
3112 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3113 } else if (Name.starts_with("avx512.kunpck")) {
3114 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3115 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3116 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3117 int Indices[64];
3118 for (unsigned i = 0; i != NumElts; ++i)
3119 Indices[i] = i;
3120
3121 // First extract half of each vector. This gives better codegen than
3122 // doing it in a single shuffle.
3123 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3124 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3125 // Concat the vectors.
3126 // NOTE: Operands have to be swapped to match intrinsic definition.
3127 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3128 Rep = Builder.CreateBitCast(Rep, CI->getType());
3129 } else if (Name == "avx512.kand.w") {
3130 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3131 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3132 Rep = Builder.CreateAnd(LHS, RHS);
3133 Rep = Builder.CreateBitCast(Rep, CI->getType());
3134 } else if (Name == "avx512.kandn.w") {
3135 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3136 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3137 LHS = Builder.CreateNot(LHS);
3138 Rep = Builder.CreateAnd(LHS, RHS);
3139 Rep = Builder.CreateBitCast(Rep, CI->getType());
3140 } else if (Name == "avx512.kor.w") {
3141 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3142 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3143 Rep = Builder.CreateOr(LHS, RHS);
3144 Rep = Builder.CreateBitCast(Rep, CI->getType());
3145 } else if (Name == "avx512.kxor.w") {
3146 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3147 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3148 Rep = Builder.CreateXor(LHS, RHS);
3149 Rep = Builder.CreateBitCast(Rep, CI->getType());
3150 } else if (Name == "avx512.kxnor.w") {
3151 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3152 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3153 LHS = Builder.CreateNot(LHS);
3154 Rep = Builder.CreateXor(LHS, RHS);
3155 Rep = Builder.CreateBitCast(Rep, CI->getType());
3156 } else if (Name == "avx512.knot.w") {
3157 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3158 Rep = Builder.CreateNot(Rep);
3159 Rep = Builder.CreateBitCast(Rep, CI->getType());
3160 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3161 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3162 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3163 Rep = Builder.CreateOr(LHS, RHS);
3164 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3165 Value *C;
3166 if (Name[14] == 'c')
3167 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3168 else
3169 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3170 Rep = Builder.CreateICmpEQ(Rep, C);
3171 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3172 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3173 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3174 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3175 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3176 Type *I32Ty = Type::getInt32Ty(C);
3177 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3178 ConstantInt::get(I32Ty, 0));
3179 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3180 ConstantInt::get(I32Ty, 0));
3181 Value *EltOp;
3182 if (Name.contains(".add."))
3183 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3184 else if (Name.contains(".sub."))
3185 EltOp = Builder.CreateFSub(Elt0, Elt1);
3186 else if (Name.contains(".mul."))
3187 EltOp = Builder.CreateFMul(Elt0, Elt1);
3188 else
3189 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3190 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3191 ConstantInt::get(I32Ty, 0));
3192 } else if (Name.starts_with("avx512.mask.pcmp")) {
3193 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3194 bool CmpEq = Name[16] == 'e';
3195 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3196 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3197 Type *OpTy = CI->getArgOperand(0)->getType();
3198 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3199 Intrinsic::ID IID;
3200 switch (VecWidth) {
3201 default:
3202 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3203 break;
3204 case 128:
3205 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3206 break;
3207 case 256:
3208 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3209 break;
3210 case 512:
3211 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3212 break;
3213 }
3214
3215 Rep =
3216 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3217 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3218 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3219 Type *OpTy = CI->getArgOperand(0)->getType();
3220 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3221 unsigned EltWidth = OpTy->getScalarSizeInBits();
3222 Intrinsic::ID IID;
3223 if (VecWidth == 128 && EltWidth == 32)
3224 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3225 else if (VecWidth == 256 && EltWidth == 32)
3226 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3227 else if (VecWidth == 512 && EltWidth == 32)
3228 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3229 else if (VecWidth == 128 && EltWidth == 64)
3230 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3231 else if (VecWidth == 256 && EltWidth == 64)
3232 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3233 else if (VecWidth == 512 && EltWidth == 64)
3234 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3235 else
3236 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3237
3238 Rep =
3239 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3240 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3241 } else if (Name.starts_with("avx512.cmp.p")) {
3242 SmallVector<Value *, 4> Args(CI->args());
3243 Type *OpTy = Args[0]->getType();
3244 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3245 unsigned EltWidth = OpTy->getScalarSizeInBits();
3246 Intrinsic::ID IID;
3247 if (VecWidth == 128 && EltWidth == 32)
3248 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3249 else if (VecWidth == 256 && EltWidth == 32)
3250 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3251 else if (VecWidth == 512 && EltWidth == 32)
3252 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3253 else if (VecWidth == 128 && EltWidth == 64)
3254 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3255 else if (VecWidth == 256 && EltWidth == 64)
3256 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3257 else if (VecWidth == 512 && EltWidth == 64)
3258 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3259 else
3260 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3261
3262 Value *Mask = Constant::getAllOnesValue(CI->getType());
3263 if (VecWidth == 512)
3264 std::swap(Mask, Args.back());
3265 Args.push_back(Mask);
3266
3267 Rep = Builder.CreateIntrinsic(IID, Args);
3268 } else if (Name.starts_with("avx512.mask.cmp.")) {
3269 // Integer compare intrinsics.
3270 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3271 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3272 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3273 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3274 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3275 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3276 Name.starts_with("avx512.cvtw2mask.") ||
3277 Name.starts_with("avx512.cvtd2mask.") ||
3278 Name.starts_with("avx512.cvtq2mask.")) {
3279 Value *Op = CI->getArgOperand(0);
3280 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3281 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3282 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3283 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3284 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3285 Name.starts_with("avx512.mask.pabs")) {
3286 Rep = upgradeAbs(Builder, *CI);
3287 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3288 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3289 Name.starts_with("avx512.mask.pmaxs")) {
3290 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3291 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3292 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3293 Name.starts_with("avx512.mask.pmaxu")) {
3294 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3295 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3296 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3297 Name.starts_with("avx512.mask.pmins")) {
3298 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3299 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3300 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3301 Name.starts_with("avx512.mask.pminu")) {
3302 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3303 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3304 Name == "avx512.pmulu.dq.512" ||
3305 Name.starts_with("avx512.mask.pmulu.dq.")) {
3306 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3307 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3308 Name == "avx512.pmul.dq.512" ||
3309 Name.starts_with("avx512.mask.pmul.dq.")) {
3310 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3311 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3312 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3313 Rep =
3314 Builder.CreateSIToFP(CI->getArgOperand(1),
3315 cast<VectorType>(CI->getType())->getElementType());
3316 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3317 } else if (Name == "avx512.cvtusi2sd") {
3318 Rep =
3319 Builder.CreateUIToFP(CI->getArgOperand(1),
3320 cast<VectorType>(CI->getType())->getElementType());
3321 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3322 } else if (Name == "sse2.cvtss2sd") {
3323 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3324 Rep = Builder.CreateFPExt(
3325 Rep, cast<VectorType>(CI->getType())->getElementType());
3326 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3327 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3328 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3329 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3330 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3331 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3332 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3333 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3334 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3335 Name == "avx512.mask.cvtqq2ps.256" ||
3336 Name == "avx512.mask.cvtqq2ps.512" ||
3337 Name == "avx512.mask.cvtuqq2ps.256" ||
3338 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3339 Name == "avx.cvt.ps2.pd.256" ||
3340 Name == "avx512.mask.cvtps2pd.128" ||
3341 Name == "avx512.mask.cvtps2pd.256") {
3342 auto *DstTy = cast<FixedVectorType>(CI->getType());
3343 Rep = CI->getArgOperand(0);
3344 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3345
3346 unsigned NumDstElts = DstTy->getNumElements();
3347 if (NumDstElts < SrcTy->getNumElements()) {
3348 assert(NumDstElts == 2 && "Unexpected vector size");
3349 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3350 }
3351
3352 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3353 bool IsUnsigned = Name.contains("cvtu");
3354 if (IsPS2PD)
3355 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3356 else if (CI->arg_size() == 4 &&
3357 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3358 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3359 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3360 : Intrinsic::x86_avx512_sitofp_round;
3361 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3362 {Rep, CI->getArgOperand(3)});
3363 } else {
3364 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3365 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3366 }
3367
3368 if (CI->arg_size() >= 3)
3369 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3370 CI->getArgOperand(1));
3371 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3372 Name.starts_with("vcvtph2ps.")) {
3373 auto *DstTy = cast<FixedVectorType>(CI->getType());
3374 Rep = CI->getArgOperand(0);
3375 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3376 unsigned NumDstElts = DstTy->getNumElements();
3377 if (NumDstElts != SrcTy->getNumElements()) {
3378 assert(NumDstElts == 4 && "Unexpected vector size");
3379 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3380 }
3381 Rep = Builder.CreateBitCast(
3382 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3383 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3384 if (CI->arg_size() >= 3)
3385 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3386 CI->getArgOperand(1));
3387 } else if (Name.starts_with("avx512.mask.load")) {
3388 // "avx512.mask.loadu." or "avx512.mask.load."
3389 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3390 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3391 CI->getArgOperand(2), Aligned);
3392 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3393 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3394 auto *PtrTy = CI->getOperand(0)->getType();
3395 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3396 ResultTy->getNumElements());
3397 Rep = Builder.CreateIntrinsic(
3398 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3399 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3400 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3401 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3402 auto *PtrTy = CI->getArgOperand(0)->getType();
3403 Value *MaskVec =
3404 getX86MaskVec(Builder, CI->getArgOperand(2),
3405 cast<FixedVectorType>(ResultTy)->getNumElements());
3406 Rep = Builder.CreateIntrinsic(
3407 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3408 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3409 } else if (Name.starts_with("avx512.mask.compress.") ||
3410 Name.starts_with("avx512.mask.expand.")) {
3411 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3412
3413 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3414 ResultTy->getNumElements());
3415
3416 bool IsCompress = Name[12] == 'c';
3417 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3418 : Intrinsic::x86_avx512_mask_expand;
3419 Rep = Builder.CreateIntrinsic(
3420 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3421 } else if (Name.starts_with("xop.vpcom")) {
3422 bool IsSigned;
3423 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3424 Name.ends_with("uq"))
3425 IsSigned = false;
3426 else if (Name.ends_with("b") || Name.ends_with("w") ||
3427 Name.ends_with("d") || Name.ends_with("q"))
3428 IsSigned = true;
3429 else
3430 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3431
3432 unsigned Imm;
3433 if (CI->arg_size() == 3) {
3434 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3435 } else {
3436 Name = Name.substr(9); // strip off "xop.vpcom"
3437 if (Name.starts_with("lt"))
3438 Imm = 0;
3439 else if (Name.starts_with("le"))
3440 Imm = 1;
3441 else if (Name.starts_with("gt"))
3442 Imm = 2;
3443 else if (Name.starts_with("ge"))
3444 Imm = 3;
3445 else if (Name.starts_with("eq"))
3446 Imm = 4;
3447 else if (Name.starts_with("ne"))
3448 Imm = 5;
3449 else if (Name.starts_with("false"))
3450 Imm = 6;
3451 else if (Name.starts_with("true"))
3452 Imm = 7;
3453 else
3454 llvm_unreachable("Unknown condition");
3455 }
3456
3457 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3458 } else if (Name.starts_with("xop.vpcmov")) {
3459 Value *Sel = CI->getArgOperand(2);
3460 Value *NotSel = Builder.CreateNot(Sel);
3461 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3462 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3463 Rep = Builder.CreateOr(Sel0, Sel1);
3464 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3465 Name.starts_with("avx512.mask.prol")) {
3466 Rep = upgradeX86Rotate(Builder, *CI, false);
3467 } else if (Name.starts_with("avx512.pror") ||
3468 Name.starts_with("avx512.mask.pror")) {
3469 Rep = upgradeX86Rotate(Builder, *CI, true);
3470 } else if (Name.starts_with("avx512.vpshld.") ||
3471 Name.starts_with("avx512.mask.vpshld") ||
3472 Name.starts_with("avx512.maskz.vpshld")) {
3473 bool ZeroMask = Name[11] == 'z';
3474 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3475 } else if (Name.starts_with("avx512.vpshrd.") ||
3476 Name.starts_with("avx512.mask.vpshrd") ||
3477 Name.starts_with("avx512.maskz.vpshrd")) {
3478 bool ZeroMask = Name[11] == 'z';
3479 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3480 } else if (Name == "sse42.crc32.64.8") {
3481 Value *Trunc0 =
3482 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3483 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3484 {Trunc0, CI->getArgOperand(1)});
3485 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3486 } else if (Name.starts_with("avx.vbroadcast.s") ||
3487 Name.starts_with("avx512.vbroadcast.s")) {
3488 // Replace broadcasts with a series of insertelements.
3489 auto *VecTy = cast<FixedVectorType>(CI->getType());
3490 Type *EltTy = VecTy->getElementType();
3491 unsigned EltNum = VecTy->getNumElements();
3492 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3493 Type *I32Ty = Type::getInt32Ty(C);
3494 Rep = PoisonValue::get(VecTy);
3495 for (unsigned I = 0; I < EltNum; ++I)
3496 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3497 } else if (Name.starts_with("sse41.pmovsx") ||
3498 Name.starts_with("sse41.pmovzx") ||
3499 Name.starts_with("avx2.pmovsx") ||
3500 Name.starts_with("avx2.pmovzx") ||
3501 Name.starts_with("avx512.mask.pmovsx") ||
3502 Name.starts_with("avx512.mask.pmovzx")) {
3503 auto *DstTy = cast<FixedVectorType>(CI->getType());
3504 unsigned NumDstElts = DstTy->getNumElements();
3505
3506 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3507 SmallVector<int, 8> ShuffleMask(NumDstElts);
3508 for (unsigned i = 0; i != NumDstElts; ++i)
3509 ShuffleMask[i] = i;
3510
3511 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3512
3513 bool DoSext = Name.contains("pmovsx");
3514 Rep =
3515 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3516 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3517 if (CI->arg_size() == 3)
3518 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3519 CI->getArgOperand(1));
3520 } else if (Name == "avx512.mask.pmov.qd.256" ||
3521 Name == "avx512.mask.pmov.qd.512" ||
3522 Name == "avx512.mask.pmov.wb.256" ||
3523 Name == "avx512.mask.pmov.wb.512") {
3524 Type *Ty = CI->getArgOperand(1)->getType();
3525 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3526 Rep =
3527 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3528 } else if (Name.starts_with("avx.vbroadcastf128") ||
3529 Name == "avx2.vbroadcasti128") {
3530 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3531 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3532 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3533 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3534 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3535 if (NumSrcElts == 2)
3536 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3537 else
3538 Rep = Builder.CreateShuffleVector(Load,
3539 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3540 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3541 Name.starts_with("avx512.mask.shuf.f")) {
3542 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3543 Type *VT = CI->getType();
3544 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3545 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3546 unsigned ControlBitsMask = NumLanes - 1;
3547 unsigned NumControlBits = NumLanes / 2;
3548 SmallVector<int, 8> ShuffleMask(0);
3549
3550 for (unsigned l = 0; l != NumLanes; ++l) {
3551 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3552 // We actually need the other source.
3553 if (l >= NumLanes / 2)
3554 LaneMask += NumLanes;
3555 for (unsigned i = 0; i != NumElementsInLane; ++i)
3556 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3557 }
3558 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3559 CI->getArgOperand(1), ShuffleMask);
3560 Rep =
3561 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3562 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3563 Name.starts_with("avx512.mask.broadcasti")) {
3564 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3565 ->getNumElements();
3566 unsigned NumDstElts =
3567 cast<FixedVectorType>(CI->getType())->getNumElements();
3568
3569 SmallVector<int, 8> ShuffleMask(NumDstElts);
3570 for (unsigned i = 0; i != NumDstElts; ++i)
3571 ShuffleMask[i] = i % NumSrcElts;
3572
3573 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3574 CI->getArgOperand(0), ShuffleMask);
3575 Rep =
3576 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3577 } else if (Name.starts_with("avx2.pbroadcast") ||
3578 Name.starts_with("avx2.vbroadcast") ||
3579 Name.starts_with("avx512.pbroadcast") ||
3580 Name.starts_with("avx512.mask.broadcast.s")) {
3581 // Replace vp?broadcasts with a vector shuffle.
3582 Value *Op = CI->getArgOperand(0);
3583 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3584 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3585 SmallVector<int, 8> M;
3586 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3587 Rep = Builder.CreateShuffleVector(Op, M);
3588
3589 if (CI->arg_size() == 3)
3590 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3591 CI->getArgOperand(1));
3592 } else if (Name.starts_with("sse2.padds.") ||
3593 Name.starts_with("avx2.padds.") ||
3594 Name.starts_with("avx512.padds.") ||
3595 Name.starts_with("avx512.mask.padds.")) {
3596 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3597 } else if (Name.starts_with("sse2.psubs.") ||
3598 Name.starts_with("avx2.psubs.") ||
3599 Name.starts_with("avx512.psubs.") ||
3600 Name.starts_with("avx512.mask.psubs.")) {
3601 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3602 } else if (Name.starts_with("sse2.paddus.") ||
3603 Name.starts_with("avx2.paddus.") ||
3604 Name.starts_with("avx512.mask.paddus.")) {
3605 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3606 } else if (Name.starts_with("sse2.psubus.") ||
3607 Name.starts_with("avx2.psubus.") ||
3608 Name.starts_with("avx512.mask.psubus.")) {
3609 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3610 } else if (Name.starts_with("avx512.mask.palignr.")) {
3611 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3612 CI->getArgOperand(1), CI->getArgOperand(2),
3613 CI->getArgOperand(3), CI->getArgOperand(4),
3614 false);
3615 } else if (Name.starts_with("avx512.mask.valign.")) {
3616 Rep = upgradeX86ALIGNIntrinsics(
3617 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3618 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3619 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3620 // 128/256-bit shift left specified in bits.
3621 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3622 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3623 Shift / 8); // Shift is in bits.
3624 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3625 // 128/256-bit shift right specified in bits.
3626 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3627 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3628 Shift / 8); // Shift is in bits.
3629 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3630 Name == "avx512.psll.dq.512") {
3631 // 128/256/512-bit shift left specified in bytes.
3632 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3633 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3634 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3635 Name == "avx512.psrl.dq.512") {
3636 // 128/256/512-bit shift right specified in bytes.
3637 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3638 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3639 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3640 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3641 Name.starts_with("avx2.pblendd.")) {
3642 Value *Op0 = CI->getArgOperand(0);
3643 Value *Op1 = CI->getArgOperand(1);
3644 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3645 auto *VecTy = cast<FixedVectorType>(CI->getType());
3646 unsigned NumElts = VecTy->getNumElements();
3647
3648 SmallVector<int, 16> Idxs(NumElts);
3649 for (unsigned i = 0; i != NumElts; ++i)
3650 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3651
3652 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3653 } else if (Name.starts_with("avx.vinsertf128.") ||
3654 Name == "avx2.vinserti128" ||
3655 Name.starts_with("avx512.mask.insert")) {
3656 Value *Op0 = CI->getArgOperand(0);
3657 Value *Op1 = CI->getArgOperand(1);
3658 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3659 unsigned DstNumElts =
3660 cast<FixedVectorType>(CI->getType())->getNumElements();
3661 unsigned SrcNumElts =
3662 cast<FixedVectorType>(Op1->getType())->getNumElements();
3663 unsigned Scale = DstNumElts / SrcNumElts;
3664
3665 // Mask off the high bits of the immediate value; hardware ignores those.
3666 Imm = Imm % Scale;
3667
3668 // Extend the second operand into a vector the size of the destination.
3669 SmallVector<int, 8> Idxs(DstNumElts);
3670 for (unsigned i = 0; i != SrcNumElts; ++i)
3671 Idxs[i] = i;
3672 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3673 Idxs[i] = SrcNumElts;
3674 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3675
3676 // Insert the second operand into the first operand.
3677
3678 // Note that there is no guarantee that instruction lowering will actually
3679 // produce a vinsertf128 instruction for the created shuffles. In
3680 // particular, the 0 immediate case involves no lane changes, so it can
3681 // be handled as a blend.
3682
3683 // Example of shuffle mask for 32-bit elements:
3684 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3685 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3686
3687 // First fill with identify mask.
3688 for (unsigned i = 0; i != DstNumElts; ++i)
3689 Idxs[i] = i;
3690 // Then replace the elements where we need to insert.
3691 for (unsigned i = 0; i != SrcNumElts; ++i)
3692 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3693 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3694
3695 // If the intrinsic has a mask operand, handle that.
3696 if (CI->arg_size() == 5)
3697 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3698 CI->getArgOperand(3));
3699 } else if (Name.starts_with("avx.vextractf128.") ||
3700 Name == "avx2.vextracti128" ||
3701 Name.starts_with("avx512.mask.vextract")) {
3702 Value *Op0 = CI->getArgOperand(0);
3703 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3704 unsigned DstNumElts =
3705 cast<FixedVectorType>(CI->getType())->getNumElements();
3706 unsigned SrcNumElts =
3707 cast<FixedVectorType>(Op0->getType())->getNumElements();
3708 unsigned Scale = SrcNumElts / DstNumElts;
3709
3710 // Mask off the high bits of the immediate value; hardware ignores those.
3711 Imm = Imm % Scale;
3712
3713 // Get indexes for the subvector of the input vector.
3714 SmallVector<int, 8> Idxs(DstNumElts);
3715 for (unsigned i = 0; i != DstNumElts; ++i) {
3716 Idxs[i] = i + (Imm * DstNumElts);
3717 }
3718 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3719
3720 // If the intrinsic has a mask operand, handle that.
3721 if (CI->arg_size() == 4)
3722 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3723 CI->getArgOperand(2));
3724 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3725 Name.starts_with("avx512.mask.perm.di.")) {
3726 Value *Op0 = CI->getArgOperand(0);
3727 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3728 auto *VecTy = cast<FixedVectorType>(CI->getType());
3729 unsigned NumElts = VecTy->getNumElements();
3730
3731 SmallVector<int, 8> Idxs(NumElts);
3732 for (unsigned i = 0; i != NumElts; ++i)
3733 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3734
3735 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3736
3737 if (CI->arg_size() == 4)
3738 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3739 CI->getArgOperand(2));
3740 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3741 // The immediate permute control byte looks like this:
3742 // [1:0] - select 128 bits from sources for low half of destination
3743 // [2] - ignore
3744 // [3] - zero low half of destination
3745 // [5:4] - select 128 bits from sources for high half of destination
3746 // [6] - ignore
3747 // [7] - zero high half of destination
3748
3749 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3750
3751 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3752 unsigned HalfSize = NumElts / 2;
3753 SmallVector<int, 8> ShuffleMask(NumElts);
3754
3755 // Determine which operand(s) are actually in use for this instruction.
3756 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3757 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3758
3759 // If needed, replace operands based on zero mask.
3760 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3761 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3762
3763 // Permute low half of result.
3764 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3765 for (unsigned i = 0; i < HalfSize; ++i)
3766 ShuffleMask[i] = StartIndex + i;
3767
3768 // Permute high half of result.
3769 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3770 for (unsigned i = 0; i < HalfSize; ++i)
3771 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3772
3773 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3774
3775 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
3776 Name.starts_with("avx512.mask.vpermil.p") ||
3777 Name.starts_with("avx512.mask.pshuf.d.")) {
3778 Value *Op0 = CI->getArgOperand(0);
3779 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3780 auto *VecTy = cast<FixedVectorType>(CI->getType());
3781 unsigned NumElts = VecTy->getNumElements();
3782 // Calculate the size of each index in the immediate.
3783 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3784 unsigned IdxMask = ((1 << IdxSize) - 1);
3785
3786 SmallVector<int, 8> Idxs(NumElts);
3787 // Lookup the bits for this element, wrapping around the immediate every
3788 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3789 // to offset by the first index of each group.
3790 for (unsigned i = 0; i != NumElts; ++i)
3791 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3792
3793 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3794
3795 if (CI->arg_size() == 4)
3796 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3797 CI->getArgOperand(2));
3798 } else if (Name == "sse2.pshufl.w" ||
3799 Name.starts_with("avx512.mask.pshufl.w.")) {
3800 Value *Op0 = CI->getArgOperand(0);
3801 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3802 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3803
3804 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
3805 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3806
3807 SmallVector<int, 16> Idxs(NumElts);
3808 for (unsigned l = 0; l != NumElts; l += 8) {
3809 for (unsigned i = 0; i != 4; ++i)
3810 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3811 for (unsigned i = 4; i != 8; ++i)
3812 Idxs[i + l] = i + l;
3813 }
3814
3815 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3816
3817 if (CI->arg_size() == 4)
3818 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3819 CI->getArgOperand(2));
3820 } else if (Name == "sse2.pshufh.w" ||
3821 Name.starts_with("avx512.mask.pshufh.w.")) {
3822 Value *Op0 = CI->getArgOperand(0);
3823 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3824 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3825
3826 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
3827 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3828
3829 SmallVector<int, 16> Idxs(NumElts);
3830 for (unsigned l = 0; l != NumElts; l += 8) {
3831 for (unsigned i = 0; i != 4; ++i)
3832 Idxs[i + l] = i + l;
3833 for (unsigned i = 0; i != 4; ++i)
3834 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3835 }
3836
3837 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3838
3839 if (CI->arg_size() == 4)
3840 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3841 CI->getArgOperand(2));
3842 } else if (Name.starts_with("avx512.mask.shuf.p")) {
3843 Value *Op0 = CI->getArgOperand(0);
3844 Value *Op1 = CI->getArgOperand(1);
3845 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3846 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3847
3848 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3849 unsigned HalfLaneElts = NumLaneElts / 2;
3850
3851 SmallVector<int, 16> Idxs(NumElts);
3852 for (unsigned i = 0; i != NumElts; ++i) {
3853 // Base index is the starting element of the lane.
3854 Idxs[i] = i - (i % NumLaneElts);
3855 // If we are half way through the lane switch to the other source.
3856 if ((i % NumLaneElts) >= HalfLaneElts)
3857 Idxs[i] += NumElts;
3858 // Now select the specific element. By adding HalfLaneElts bits from
3859 // the immediate. Wrapping around the immediate every 8-bits.
3860 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3861 }
3862
3863 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3864
3865 Rep =
3866 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3867 } else if (Name.starts_with("avx512.mask.movddup") ||
3868 Name.starts_with("avx512.mask.movshdup") ||
3869 Name.starts_with("avx512.mask.movsldup")) {
3870 Value *Op0 = CI->getArgOperand(0);
3871 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3872 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3873
3874 unsigned Offset = 0;
3875 if (Name.starts_with("avx512.mask.movshdup."))
3876 Offset = 1;
3877
3878 SmallVector<int, 16> Idxs(NumElts);
3879 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3880 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3881 Idxs[i + l + 0] = i + l + Offset;
3882 Idxs[i + l + 1] = i + l + Offset;
3883 }
3884
3885 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3886
3887 Rep =
3888 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3889 } else if (Name.starts_with("avx512.mask.punpckl") ||
3890 Name.starts_with("avx512.mask.unpckl.")) {
3891 Value *Op0 = CI->getArgOperand(0);
3892 Value *Op1 = CI->getArgOperand(1);
3893 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3894 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3895
3896 SmallVector<int, 64> Idxs(NumElts);
3897 for (int l = 0; l != NumElts; l += NumLaneElts)
3898 for (int i = 0; i != NumLaneElts; ++i)
3899 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3900
3901 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3902
3903 Rep =
3904 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3905 } else if (Name.starts_with("avx512.mask.punpckh") ||
3906 Name.starts_with("avx512.mask.unpckh.")) {
3907 Value *Op0 = CI->getArgOperand(0);
3908 Value *Op1 = CI->getArgOperand(1);
3909 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3910 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3911
3912 SmallVector<int, 64> Idxs(NumElts);
3913 for (int l = 0; l != NumElts; l += NumLaneElts)
3914 for (int i = 0; i != NumLaneElts; ++i)
3915 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3916
3917 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3918
3919 Rep =
3920 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3921 } else if (Name.starts_with("avx512.mask.and.") ||
3922 Name.starts_with("avx512.mask.pand.")) {
3923 VectorType *FTy = cast<VectorType>(CI->getType());
3924 VectorType *ITy = VectorType::getInteger(FTy);
3925 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3926 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3927 Rep = Builder.CreateBitCast(Rep, FTy);
3928 Rep =
3929 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3930 } else if (Name.starts_with("avx512.mask.andn.") ||
3931 Name.starts_with("avx512.mask.pandn.")) {
3932 VectorType *FTy = cast<VectorType>(CI->getType());
3933 VectorType *ITy = VectorType::getInteger(FTy);
3934 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3935 Rep = Builder.CreateAnd(Rep,
3936 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3937 Rep = Builder.CreateBitCast(Rep, FTy);
3938 Rep =
3939 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3940 } else if (Name.starts_with("avx512.mask.or.") ||
3941 Name.starts_with("avx512.mask.por.")) {
3942 VectorType *FTy = cast<VectorType>(CI->getType());
3943 VectorType *ITy = VectorType::getInteger(FTy);
3944 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3945 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3946 Rep = Builder.CreateBitCast(Rep, FTy);
3947 Rep =
3948 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3949 } else if (Name.starts_with("avx512.mask.xor.") ||
3950 Name.starts_with("avx512.mask.pxor.")) {
3951 VectorType *FTy = cast<VectorType>(CI->getType());
3952 VectorType *ITy = VectorType::getInteger(FTy);
3953 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3954 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3955 Rep = Builder.CreateBitCast(Rep, FTy);
3956 Rep =
3957 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3958 } else if (Name.starts_with("avx512.mask.padd.")) {
3959 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3960 Rep =
3961 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3962 } else if (Name.starts_with("avx512.mask.psub.")) {
3963 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3964 Rep =
3965 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3966 } else if (Name.starts_with("avx512.mask.pmull.")) {
3967 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3968 Rep =
3969 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3970 } else if (Name.starts_with("avx512.mask.add.p")) {
3971 if (Name.ends_with(".512")) {
3972 Intrinsic::ID IID;
3973 if (Name[17] == 's')
3974 IID = Intrinsic::x86_avx512_add_ps_512;
3975 else
3976 IID = Intrinsic::x86_avx512_add_pd_512;
3977
3978 Rep = Builder.CreateIntrinsic(
3979 IID,
3980 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3981 } else {
3982 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3983 }
3984 Rep =
3985 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3986 } else if (Name.starts_with("avx512.mask.div.p")) {
3987 if (Name.ends_with(".512")) {
3988 Intrinsic::ID IID;
3989 if (Name[17] == 's')
3990 IID = Intrinsic::x86_avx512_div_ps_512;
3991 else
3992 IID = Intrinsic::x86_avx512_div_pd_512;
3993
3994 Rep = Builder.CreateIntrinsic(
3995 IID,
3996 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3997 } else {
3998 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3999 }
4000 Rep =
4001 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4002 } else if (Name.starts_with("avx512.mask.mul.p")) {
4003 if (Name.ends_with(".512")) {
4004 Intrinsic::ID IID;
4005 if (Name[17] == 's')
4006 IID = Intrinsic::x86_avx512_mul_ps_512;
4007 else
4008 IID = Intrinsic::x86_avx512_mul_pd_512;
4009
4010 Rep = Builder.CreateIntrinsic(
4011 IID,
4012 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4013 } else {
4014 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4015 }
4016 Rep =
4017 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4018 } else if (Name.starts_with("avx512.mask.sub.p")) {
4019 if (Name.ends_with(".512")) {
4020 Intrinsic::ID IID;
4021 if (Name[17] == 's')
4022 IID = Intrinsic::x86_avx512_sub_ps_512;
4023 else
4024 IID = Intrinsic::x86_avx512_sub_pd_512;
4025
4026 Rep = Builder.CreateIntrinsic(
4027 IID,
4028 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4029 } else {
4030 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4031 }
4032 Rep =
4033 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4034 } else if ((Name.starts_with("avx512.mask.max.p") ||
4035 Name.starts_with("avx512.mask.min.p")) &&
4036 Name.drop_front(18) == ".512") {
4037 bool IsDouble = Name[17] == 'd';
4038 bool IsMin = Name[13] == 'i';
4039 static const Intrinsic::ID MinMaxTbl[2][2] = {
4040 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4041 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4042 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4043
4044 Rep = Builder.CreateIntrinsic(
4045 IID,
4046 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4047 Rep =
4048 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4049 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4050 Rep =
4051 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4052 {CI->getArgOperand(0), Builder.getInt1(false)});
4053 Rep =
4054 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4055 } else if (Name.starts_with("avx512.mask.psll")) {
4056 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4057 bool IsVariable = Name[16] == 'v';
4058 char Size = Name[16] == '.' ? Name[17]
4059 : Name[17] == '.' ? Name[18]
4060 : Name[18] == '.' ? Name[19]
4061 : Name[20];
4062
4063 Intrinsic::ID IID;
4064 if (IsVariable && Name[17] != '.') {
4065 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4066 IID = Intrinsic::x86_avx2_psllv_q;
4067 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4068 IID = Intrinsic::x86_avx2_psllv_q_256;
4069 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4070 IID = Intrinsic::x86_avx2_psllv_d;
4071 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4072 IID = Intrinsic::x86_avx2_psllv_d_256;
4073 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4074 IID = Intrinsic::x86_avx512_psllv_w_128;
4075 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4076 IID = Intrinsic::x86_avx512_psllv_w_256;
4077 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4078 IID = Intrinsic::x86_avx512_psllv_w_512;
4079 else
4080 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4081 } else if (Name.ends_with(".128")) {
4082 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4083 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4084 : Intrinsic::x86_sse2_psll_d;
4085 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4086 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4087 : Intrinsic::x86_sse2_psll_q;
4088 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4089 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4090 : Intrinsic::x86_sse2_psll_w;
4091 else
4092 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4093 } else if (Name.ends_with(".256")) {
4094 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4095 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4096 : Intrinsic::x86_avx2_psll_d;
4097 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4098 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4099 : Intrinsic::x86_avx2_psll_q;
4100 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4101 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4102 : Intrinsic::x86_avx2_psll_w;
4103 else
4104 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4105 } else {
4106 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4107 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4108 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4109 : Intrinsic::x86_avx512_psll_d_512;
4110 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4111 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4112 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4113 : Intrinsic::x86_avx512_psll_q_512;
4114 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4115 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4116 : Intrinsic::x86_avx512_psll_w_512;
4117 else
4118 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4119 }
4120
4121 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4122 } else if (Name.starts_with("avx512.mask.psrl")) {
4123 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4124 bool IsVariable = Name[16] == 'v';
4125 char Size = Name[16] == '.' ? Name[17]
4126 : Name[17] == '.' ? Name[18]
4127 : Name[18] == '.' ? Name[19]
4128 : Name[20];
4129
4130 Intrinsic::ID IID;
4131 if (IsVariable && Name[17] != '.') {
4132 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4133 IID = Intrinsic::x86_avx2_psrlv_q;
4134 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4135 IID = Intrinsic::x86_avx2_psrlv_q_256;
4136 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4137 IID = Intrinsic::x86_avx2_psrlv_d;
4138 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4139 IID = Intrinsic::x86_avx2_psrlv_d_256;
4140 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4141 IID = Intrinsic::x86_avx512_psrlv_w_128;
4142 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4143 IID = Intrinsic::x86_avx512_psrlv_w_256;
4144 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4145 IID = Intrinsic::x86_avx512_psrlv_w_512;
4146 else
4147 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4148 } else if (Name.ends_with(".128")) {
4149 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4150 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4151 : Intrinsic::x86_sse2_psrl_d;
4152 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4153 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4154 : Intrinsic::x86_sse2_psrl_q;
4155 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4156 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4157 : Intrinsic::x86_sse2_psrl_w;
4158 else
4159 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4160 } else if (Name.ends_with(".256")) {
4161 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4162 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4163 : Intrinsic::x86_avx2_psrl_d;
4164 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4165 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4166 : Intrinsic::x86_avx2_psrl_q;
4167 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4168 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4169 : Intrinsic::x86_avx2_psrl_w;
4170 else
4171 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4172 } else {
4173 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4174 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4175 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4176 : Intrinsic::x86_avx512_psrl_d_512;
4177 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4178 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4179 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4180 : Intrinsic::x86_avx512_psrl_q_512;
4181 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4182 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4183 : Intrinsic::x86_avx512_psrl_w_512;
4184 else
4185 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4186 }
4187
4188 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4189 } else if (Name.starts_with("avx512.mask.psra")) {
4190 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4191 bool IsVariable = Name[16] == 'v';
4192 char Size = Name[16] == '.' ? Name[17]
4193 : Name[17] == '.' ? Name[18]
4194 : Name[18] == '.' ? Name[19]
4195 : Name[20];
4196
4197 Intrinsic::ID IID;
4198 if (IsVariable && Name[17] != '.') {
4199 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4200 IID = Intrinsic::x86_avx2_psrav_d;
4201 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4202 IID = Intrinsic::x86_avx2_psrav_d_256;
4203 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4204 IID = Intrinsic::x86_avx512_psrav_w_128;
4205 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4206 IID = Intrinsic::x86_avx512_psrav_w_256;
4207 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4208 IID = Intrinsic::x86_avx512_psrav_w_512;
4209 else
4210 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4211 } else if (Name.ends_with(".128")) {
4212 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4213 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4214 : Intrinsic::x86_sse2_psra_d;
4215 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4216 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4217 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4218 : Intrinsic::x86_avx512_psra_q_128;
4219 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4220 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4221 : Intrinsic::x86_sse2_psra_w;
4222 else
4223 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4224 } else if (Name.ends_with(".256")) {
4225 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4226 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4227 : Intrinsic::x86_avx2_psra_d;
4228 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4229 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4230 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4231 : Intrinsic::x86_avx512_psra_q_256;
4232 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4233 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4234 : Intrinsic::x86_avx2_psra_w;
4235 else
4236 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4237 } else {
4238 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4239 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4240 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4241 : Intrinsic::x86_avx512_psra_d_512;
4242 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4243 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4244 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4245 : Intrinsic::x86_avx512_psra_q_512;
4246 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4247 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4248 : Intrinsic::x86_avx512_psra_w_512;
4249 else
4250 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4251 }
4252
4253 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4254 } else if (Name.starts_with("avx512.mask.move.s")) {
4255 Rep = upgradeMaskedMove(Builder, *CI);
4256 } else if (Name.starts_with("avx512.cvtmask2")) {
4257 Rep = upgradeMaskToInt(Builder, *CI);
4258 } else if (Name.ends_with(".movntdqa")) {
4259 MDNode *Node = MDNode::get(
4260 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4261
4262 LoadInst *LI = Builder.CreateAlignedLoad(
4263 CI->getType(), CI->getArgOperand(0),
4264 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4265 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4266 Rep = LI;
4267 } else if (Name.starts_with("fma.vfmadd.") ||
4268 Name.starts_with("fma.vfmsub.") ||
4269 Name.starts_with("fma.vfnmadd.") ||
4270 Name.starts_with("fma.vfnmsub.")) {
4271 bool NegMul = Name[6] == 'n';
4272 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4273 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4274
4275 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4276 CI->getArgOperand(2)};
4277
4278 if (IsScalar) {
4279 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4280 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4281 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4282 }
4283
4284 if (NegMul && !IsScalar)
4285 Ops[0] = Builder.CreateFNeg(Ops[0]);
4286 if (NegMul && IsScalar)
4287 Ops[1] = Builder.CreateFNeg(Ops[1]);
4288 if (NegAcc)
4289 Ops[2] = Builder.CreateFNeg(Ops[2]);
4290
4291 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4292
4293 if (IsScalar)
4294 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4295 } else if (Name.starts_with("fma4.vfmadd.s")) {
4296 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4297 CI->getArgOperand(2)};
4298
4299 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4300 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4301 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4302
4303 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4304
4305 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4306 Rep, (uint64_t)0);
4307 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4308 Name.starts_with("avx512.maskz.vfmadd.s") ||
4309 Name.starts_with("avx512.mask3.vfmadd.s") ||
4310 Name.starts_with("avx512.mask3.vfmsub.s") ||
4311 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4312 bool IsMask3 = Name[11] == '3';
4313 bool IsMaskZ = Name[11] == 'z';
4314 // Drop the "avx512.mask." to make it easier.
4315 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4316 bool NegMul = Name[2] == 'n';
4317 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4318
4319 Value *A = CI->getArgOperand(0);
4320 Value *B = CI->getArgOperand(1);
4321 Value *C = CI->getArgOperand(2);
4322
4323 if (NegMul && (IsMask3 || IsMaskZ))
4324 A = Builder.CreateFNeg(A);
4325 if (NegMul && !(IsMask3 || IsMaskZ))
4326 B = Builder.CreateFNeg(B);
4327 if (NegAcc)
4328 C = Builder.CreateFNeg(C);
4329
4330 A = Builder.CreateExtractElement(A, (uint64_t)0);
4331 B = Builder.CreateExtractElement(B, (uint64_t)0);
4332 C = Builder.CreateExtractElement(C, (uint64_t)0);
4333
4334 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4335 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4336 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4337
4338 Intrinsic::ID IID;
4339 if (Name.back() == 'd')
4340 IID = Intrinsic::x86_avx512_vfmadd_f64;
4341 else
4342 IID = Intrinsic::x86_avx512_vfmadd_f32;
4343 Rep = Builder.CreateIntrinsic(IID, Ops);
4344 } else {
4345 Rep = Builder.CreateFMA(A, B, C);
4346 }
4347
4348 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4349 : IsMask3 ? C
4350 : A;
4351
4352 // For Mask3 with NegAcc, we need to create a new extractelement that
4353 // avoids the negation above.
4354 if (NegAcc && IsMask3)
4355 PassThru =
4356 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4357
4358 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4359 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4360 (uint64_t)0);
4361 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4362 Name.starts_with("avx512.mask.vfnmadd.p") ||
4363 Name.starts_with("avx512.mask.vfnmsub.p") ||
4364 Name.starts_with("avx512.mask3.vfmadd.p") ||
4365 Name.starts_with("avx512.mask3.vfmsub.p") ||
4366 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4367 Name.starts_with("avx512.maskz.vfmadd.p")) {
4368 bool IsMask3 = Name[11] == '3';
4369 bool IsMaskZ = Name[11] == 'z';
4370 // Drop the "avx512.mask." to make it easier.
4371 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4372 bool NegMul = Name[2] == 'n';
4373 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4374
4375 Value *A = CI->getArgOperand(0);
4376 Value *B = CI->getArgOperand(1);
4377 Value *C = CI->getArgOperand(2);
4378
4379 if (NegMul && (IsMask3 || IsMaskZ))
4380 A = Builder.CreateFNeg(A);
4381 if (NegMul && !(IsMask3 || IsMaskZ))
4382 B = Builder.CreateFNeg(B);
4383 if (NegAcc)
4384 C = Builder.CreateFNeg(C);
4385
4386 if (CI->arg_size() == 5 &&
4387 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4388 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4389 Intrinsic::ID IID;
4390 // Check the character before ".512" in string.
4391 if (Name[Name.size() - 5] == 's')
4392 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4393 else
4394 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4395
4396 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4397 } else {
4398 Rep = Builder.CreateFMA(A, B, C);
4399 }
4400
4401 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4402 : IsMask3 ? CI->getArgOperand(2)
4403 : CI->getArgOperand(0);
4404
4405 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4406 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4407 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4408 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4409 Intrinsic::ID IID;
4410 if (VecWidth == 128 && EltWidth == 32)
4411 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4412 else if (VecWidth == 256 && EltWidth == 32)
4413 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4414 else if (VecWidth == 128 && EltWidth == 64)
4415 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4416 else if (VecWidth == 256 && EltWidth == 64)
4417 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4418 else
4419 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4420
4421 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4422 CI->getArgOperand(2)};
4423 Ops[2] = Builder.CreateFNeg(Ops[2]);
4424 Rep = Builder.CreateIntrinsic(IID, Ops);
4425 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4426 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4427 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4428 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4429 bool IsMask3 = Name[11] == '3';
4430 bool IsMaskZ = Name[11] == 'z';
4431 // Drop the "avx512.mask." to make it easier.
4432 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4433 bool IsSubAdd = Name[3] == 's';
4434 if (CI->arg_size() == 5) {
4435 Intrinsic::ID IID;
4436 // Check the character before ".512" in string.
4437 if (Name[Name.size() - 5] == 's')
4438 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4439 else
4440 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4441
4442 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4443 CI->getArgOperand(2), CI->getArgOperand(4)};
4444 if (IsSubAdd)
4445 Ops[2] = Builder.CreateFNeg(Ops[2]);
4446
4447 Rep = Builder.CreateIntrinsic(IID, Ops);
4448 } else {
4449 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4450
4451 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4452 CI->getArgOperand(2)};
4453
4454 Function *FMA = Intrinsic::getOrInsertDeclaration(
4455 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4456 Value *Odd = Builder.CreateCall(FMA, Ops);
4457 Ops[2] = Builder.CreateFNeg(Ops[2]);
4458 Value *Even = Builder.CreateCall(FMA, Ops);
4459
4460 if (IsSubAdd)
4461 std::swap(Even, Odd);
4462
4463 SmallVector<int, 32> Idxs(NumElts);
4464 for (int i = 0; i != NumElts; ++i)
4465 Idxs[i] = i + (i % 2) * NumElts;
4466
4467 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4468 }
4469
4470 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4471 : IsMask3 ? CI->getArgOperand(2)
4472 : CI->getArgOperand(0);
4473
4474 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4475 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4476 Name.starts_with("avx512.maskz.pternlog.")) {
4477 bool ZeroMask = Name[11] == 'z';
4478 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4479 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4480 Intrinsic::ID IID;
4481 if (VecWidth == 128 && EltWidth == 32)
4482 IID = Intrinsic::x86_avx512_pternlog_d_128;
4483 else if (VecWidth == 256 && EltWidth == 32)
4484 IID = Intrinsic::x86_avx512_pternlog_d_256;
4485 else if (VecWidth == 512 && EltWidth == 32)
4486 IID = Intrinsic::x86_avx512_pternlog_d_512;
4487 else if (VecWidth == 128 && EltWidth == 64)
4488 IID = Intrinsic::x86_avx512_pternlog_q_128;
4489 else if (VecWidth == 256 && EltWidth == 64)
4490 IID = Intrinsic::x86_avx512_pternlog_q_256;
4491 else if (VecWidth == 512 && EltWidth == 64)
4492 IID = Intrinsic::x86_avx512_pternlog_q_512;
4493 else
4494 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4495
4496 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4497 CI->getArgOperand(2), CI->getArgOperand(3)};
4498 Rep = Builder.CreateIntrinsic(IID, Args);
4499 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4500 : CI->getArgOperand(0);
4501 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4502 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4503 Name.starts_with("avx512.maskz.vpmadd52")) {
4504 bool ZeroMask = Name[11] == 'z';
4505 bool High = Name[20] == 'h' || Name[21] == 'h';
4506 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4507 Intrinsic::ID IID;
4508 if (VecWidth == 128 && !High)
4509 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4510 else if (VecWidth == 256 && !High)
4511 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4512 else if (VecWidth == 512 && !High)
4513 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4514 else if (VecWidth == 128 && High)
4515 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4516 else if (VecWidth == 256 && High)
4517 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4518 else if (VecWidth == 512 && High)
4519 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4520 else
4521 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4522
4523 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4524 CI->getArgOperand(2)};
4525 Rep = Builder.CreateIntrinsic(IID, Args);
4526 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4527 : CI->getArgOperand(0);
4528 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4529 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4530 Name.starts_with("avx512.mask.vpermt2var.") ||
4531 Name.starts_with("avx512.maskz.vpermt2var.")) {
4532 bool ZeroMask = Name[11] == 'z';
4533 bool IndexForm = Name[17] == 'i';
4534 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4535 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4536 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4537 Name.starts_with("avx512.mask.vpdpbusds.") ||
4538 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4539 bool ZeroMask = Name[11] == 'z';
4540 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4541 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4542 Intrinsic::ID IID;
4543 if (VecWidth == 128 && !IsSaturating)
4544 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4545 else if (VecWidth == 256 && !IsSaturating)
4546 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4547 else if (VecWidth == 512 && !IsSaturating)
4548 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4549 else if (VecWidth == 128 && IsSaturating)
4550 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4551 else if (VecWidth == 256 && IsSaturating)
4552 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4553 else if (VecWidth == 512 && IsSaturating)
4554 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4555 else
4556 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4557
4558 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4559 CI->getArgOperand(2)};
4560
4561 // Input arguments types were incorrectly set to vectors of i32 before but
4562 // they should be vectors of i8. Insert bit cast when encountering the old
4563 // types
4564 if (Args[1]->getType()->isVectorTy() &&
4565 cast<VectorType>(Args[1]->getType())
4566 ->getElementType()
4567 ->isIntegerTy(32) &&
4568 Args[2]->getType()->isVectorTy() &&
4569 cast<VectorType>(Args[2]->getType())
4570 ->getElementType()
4571 ->isIntegerTy(32)) {
4572 Type *NewArgType = nullptr;
4573 if (VecWidth == 128)
4574 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4575 else if (VecWidth == 256)
4576 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4577 else if (VecWidth == 512)
4578 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4579 else
4580 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4581 CI);
4582
4583 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4584 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4585 }
4586
4587 Rep = Builder.CreateIntrinsic(IID, Args);
4588 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4589 : CI->getArgOperand(0);
4590 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4591 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4592 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4593 Name.starts_with("avx512.mask.vpdpwssds.") ||
4594 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4595 bool ZeroMask = Name[11] == 'z';
4596 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4597 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4598 Intrinsic::ID IID;
4599 if (VecWidth == 128 && !IsSaturating)
4600 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4601 else if (VecWidth == 256 && !IsSaturating)
4602 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4603 else if (VecWidth == 512 && !IsSaturating)
4604 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4605 else if (VecWidth == 128 && IsSaturating)
4606 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4607 else if (VecWidth == 256 && IsSaturating)
4608 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4609 else if (VecWidth == 512 && IsSaturating)
4610 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4611 else
4612 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4613
4614 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4615 CI->getArgOperand(2)};
4616
4617 // Input arguments types were incorrectly set to vectors of i32 before but
4618 // they should be vectors of i16. Insert bit cast when encountering the old
4619 // types
4620 if (Args[1]->getType()->isVectorTy() &&
4621 cast<VectorType>(Args[1]->getType())
4622 ->getElementType()
4623 ->isIntegerTy(32) &&
4624 Args[2]->getType()->isVectorTy() &&
4625 cast<VectorType>(Args[2]->getType())
4626 ->getElementType()
4627 ->isIntegerTy(32)) {
4628 Type *NewArgType = nullptr;
4629 if (VecWidth == 128)
4630 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4631 else if (VecWidth == 256)
4632 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4633 else if (VecWidth == 512)
4634 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4635 else
4636 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4637 CI);
4638
4639 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4640 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4641 }
4642
4643 Rep = Builder.CreateIntrinsic(IID, Args);
4644 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4645 : CI->getArgOperand(0);
4646 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4647 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4648 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4649 Name == "subborrow.u32" || Name == "subborrow.u64") {
4650 Intrinsic::ID IID;
4651 if (Name[0] == 'a' && Name.back() == '2')
4652 IID = Intrinsic::x86_addcarry_32;
4653 else if (Name[0] == 'a' && Name.back() == '4')
4654 IID = Intrinsic::x86_addcarry_64;
4655 else if (Name[0] == 's' && Name.back() == '2')
4656 IID = Intrinsic::x86_subborrow_32;
4657 else if (Name[0] == 's' && Name.back() == '4')
4658 IID = Intrinsic::x86_subborrow_64;
4659 else
4660 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4661
4662 // Make a call with 3 operands.
4663 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4664 CI->getArgOperand(2)};
4665 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4666
4667 // Extract the second result and store it.
4668 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4669 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4670 // Replace the original call result with the first result of the new call.
4671 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4672
4673 CI->replaceAllUsesWith(CF);
4674 Rep = nullptr;
4675 } else if (Name.starts_with("avx512.mask.") &&
4676 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4677 // Rep will be updated by the call in the condition.
4678 } else if (Name.starts_with("bmi.pdep.")) {
4679 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
4680 } else if (Name.starts_with("bmi.pext.")) {
4681 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
4682 } else
4683 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4684
4685 return Rep;
4686}
4687
4688static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4689 Function *F, IRBuilder<> &Builder) {
4690 if (Name.starts_with("neon.bfcvt")) {
4691 if (Name.starts_with("neon.bfcvtn2")) {
4692 SmallVector<int, 32> LoMask(4);
4693 std::iota(LoMask.begin(), LoMask.end(), 0);
4694 SmallVector<int, 32> ConcatMask(8);
4695 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4696 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4697 Value *Trunc =
4698 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4699 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4700 } else if (Name.starts_with("neon.bfcvtn")) {
4701 SmallVector<int, 32> ConcatMask(8);
4702 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4703 Type *V4BF16 =
4704 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4705 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4706 dbgs() << "Trunc: " << *Trunc << "\n";
4707 return Builder.CreateShuffleVector(
4708 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4709 } else {
4710 return Builder.CreateFPTrunc(CI->getOperand(0),
4711 Type::getBFloatTy(F->getContext()));
4712 }
4713 } else if (Name.starts_with("sve.fcvt")) {
4714 Intrinsic::ID NewID =
4715 StringSwitch<Intrinsic::ID>(Name)
4716 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4717 .Case("sve.fcvtnt.bf16f32",
4718 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4719 .Default(Intrinsic::not_intrinsic);
4720 if (NewID == Intrinsic::not_intrinsic)
4721 llvm_unreachable("Unhandled Intrinsic!");
4722
4723 SmallVector<Value *, 3> Args(CI->args());
4724
4725 // The original intrinsics incorrectly used a predicate based on the
4726 // smallest element type rather than the largest.
4727 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4728 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4729
4730 if (Args[1]->getType() != BadPredTy)
4731 llvm_unreachable("Unexpected predicate type!");
4732
4733 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4734 BadPredTy, Args[1]);
4735 Args[1] = Builder.CreateIntrinsic(
4736 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4737
4738 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
4739 CI->getName());
4740 }
4741
4742 if (Name == "neon.vcvtfp2hf")
4743 return Builder.CreateBitCast(
4744 Builder.CreateFPTrunc(
4745 CI->getOperand(0),
4746 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4747 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
4748 if (Name == "neon.vcvthf2fp")
4749 return Builder.CreateFPExt(
4750 Builder.CreateBitCast(
4751 CI->getOperand(0),
4752 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4753 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
4754
4755 llvm_unreachable("Unhandled Intrinsic!");
4756}
4757
4758static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
4759 IRBuilder<> &Builder) {
4760 if (Name == "mve.vctp64.old") {
4761 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
4762 // correct type.
4763 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
4764 CI->getArgOperand(0),
4765 /*FMFSource=*/nullptr, CI->getName());
4766 Value *C1 = Builder.CreateIntrinsic(
4767 Intrinsic::arm_mve_pred_v2i,
4768 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
4769 return Builder.CreateIntrinsic(
4770 Intrinsic::arm_mve_pred_i2v,
4771 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
4772 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4773 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4774 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4775 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4776 Name ==
4777 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4778 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4779 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
4780 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
4781 Name ==
4782 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
4783 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
4784 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
4785 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
4786 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
4787 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
4788 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
4789 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
4790 std::vector<Type *> Tys;
4791 unsigned ID = CI->getIntrinsicID();
4792 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
4793 switch (ID) {
4794 case Intrinsic::arm_mve_mull_int_predicated:
4795 case Intrinsic::arm_mve_vqdmull_predicated:
4796 case Intrinsic::arm_mve_vldr_gather_base_predicated:
4797 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
4798 break;
4799 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4800 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4801 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4802 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
4803 V2I1Ty};
4804 break;
4805 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4806 Tys = {CI->getType(), CI->getOperand(0)->getType(),
4807 CI->getOperand(1)->getType(), V2I1Ty};
4808 break;
4809 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4810 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
4811 CI->getOperand(2)->getType(), V2I1Ty};
4812 break;
4813 case Intrinsic::arm_cde_vcx1q_predicated:
4814 case Intrinsic::arm_cde_vcx1qa_predicated:
4815 case Intrinsic::arm_cde_vcx2q_predicated:
4816 case Intrinsic::arm_cde_vcx2qa_predicated:
4817 case Intrinsic::arm_cde_vcx3q_predicated:
4818 case Intrinsic::arm_cde_vcx3qa_predicated:
4819 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
4820 break;
4821 default:
4822 llvm_unreachable("Unhandled Intrinsic!");
4823 }
4824
4825 std::vector<Value *> Ops;
4826 for (Value *Op : CI->args()) {
4827 Type *Ty = Op->getType();
4828 if (Ty->getScalarSizeInBits() == 1) {
4829 Value *C1 = Builder.CreateIntrinsic(
4830 Intrinsic::arm_mve_pred_v2i,
4831 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
4832 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
4833 }
4834 Ops.push_back(Op);
4835 }
4836
4837 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
4838 CI->getName());
4839 }
4840 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
4841}
4842
4843// These are expected to have the arguments:
4844// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
4845//
4846// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
4847//
4848static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
4849 Function *F, IRBuilder<> &Builder) {
4850 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
4851 // for compatibility.
4852 auto UpgradeLegacyWMMAIUIntrinsicCall =
4853 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
4854 ArrayRef<Type *> OverloadTys) -> Value * {
4855 // Prepare arguments, append clamp=0 for compatibility
4856 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
4857 Args.push_back(Builder.getFalse());
4858
4859 // Insert the declaration for the right overload types
4860 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4861 F->getParent(), F->getIntrinsicID(), OverloadTys);
4862
4863 // Copy operand bundles if any
4864 SmallVector<OperandBundleDef, 1> Bundles;
4865 CI->getOperandBundlesAsDefs(Bundles);
4866
4867 // Create the new call and copy calling properties
4868 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4869 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4870 NewCall->setCallingConv(CI->getCallingConv());
4871 NewCall->setAttributes(CI->getAttributes());
4872 NewCall->setDebugLoc(CI->getDebugLoc());
4873 NewCall->copyMetadata(*CI);
4874 return NewCall;
4875 };
4876
4877 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
4878 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
4879 "intrinsic should have 7 arguments");
4880 Type *T1 = CI->getArgOperand(4)->getType();
4881 Type *T2 = CI->getArgOperand(1)->getType();
4882 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
4883 }
4884 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
4885 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
4886 "intrinsic should have 8 arguments");
4887 Type *T1 = CI->getArgOperand(4)->getType();
4888 Type *T2 = CI->getArgOperand(1)->getType();
4889 Type *T3 = CI->getArgOperand(3)->getType();
4890 Type *T4 = CI->getArgOperand(5)->getType();
4891 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
4892 }
4893
4894 switch (F->getIntrinsicID()) {
4895 default:
4896 break;
4897 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4898 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4899 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4900 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4901 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4902 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
4903 // Drop src0 and src1 modifiers.
4904 const Value *Op0 = CI->getArgOperand(0);
4905 const Value *Op2 = CI->getArgOperand(2);
4906 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
4907 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
4908 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
4909 if (!ModA->isZero() || !ModB->isZero())
4910 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
4911
4912 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
4913 for (int I = 4, E = CI->arg_size(); I < E; ++I)
4914 Args.push_back(CI->getArgOperand(I));
4915
4916 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
4917 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
4918 Overloads.push_back(Args[3]->getType());
4919 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4920 F->getParent(), F->getIntrinsicID(), Overloads);
4921
4922 SmallVector<OperandBundleDef, 1> Bundles;
4923 CI->getOperandBundlesAsDefs(Bundles);
4924
4925 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4926 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4927 NewCall->setCallingConv(CI->getCallingConv());
4928 NewCall->setAttributes(CI->getAttributes());
4929 NewCall->setDebugLoc(CI->getDebugLoc());
4930 NewCall->copyMetadata(*CI);
4931 NewCall->takeName(CI);
4932 return NewCall;
4933 }
4934 }
4935
4936 AtomicRMWInst::BinOp RMWOp =
4937 StringSwitch<AtomicRMWInst::BinOp>(Name)
4938 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
4939 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
4940 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
4941 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
4942 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
4943 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
4944 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
4945 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
4946 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
4947 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
4948 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
4949 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
4950 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
4951
4952 unsigned NumOperands = CI->getNumOperands();
4953 if (NumOperands < 3) // Malformed bitcode.
4954 return nullptr;
4955
4956 Value *Ptr = CI->getArgOperand(0);
4957 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
4958 if (!PtrTy) // Malformed.
4959 return nullptr;
4960
4961 Value *Val = CI->getArgOperand(1);
4962 if (Val->getType() != CI->getType()) // Malformed.
4963 return nullptr;
4964
4965 ConstantInt *OrderArg = nullptr;
4966 bool IsVolatile = false;
4967
4968 // These should have 5 arguments (plus the callee). A separate version of the
4969 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
4970 if (NumOperands > 3)
4971 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
4972
4973 // Ignore scope argument at 3
4974
4975 if (NumOperands > 5) {
4976 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
4977 IsVolatile = !VolatileArg || !VolatileArg->isZero();
4978 }
4979
4980 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
4981 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
4982 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
4983 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
4984 Order = AtomicOrdering::SequentiallyConsistent;
4985
4986 LLVMContext &Ctx = F->getContext();
4987
4988 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
4989 Type *RetTy = CI->getType();
4990 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
4991 if (VT->getElementType()->isIntegerTy(16)) {
4992 VectorType *AsBF16 =
4993 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
4994 Val = Builder.CreateBitCast(Val, AsBF16);
4995 }
4996 }
4997
4998 // The scope argument never really worked correctly. Use agent as the most
4999 // conservative option which should still always produce the instruction.
5000 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5001 AtomicRMWInst *RMW =
5002 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5003
5004 unsigned AddrSpace = PtrTy->getAddressSpace();
5005 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5006 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5007 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5008 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5009 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5010 }
5011
5012 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5013 MDBuilder MDB(F->getContext());
5014 MDNode *RangeNotPrivate =
5015 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5016 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5017 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5018 }
5019
5020 if (IsVolatile)
5021 RMW->setVolatile(true);
5022
5023 return Builder.CreateBitCast(RMW, RetTy);
5024}
5025
5026/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5027/// plain MDNode, as it's the verifier's job to check these are the correct
5028/// types later.
5029static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5030 if (Op < CI->arg_size()) {
5031 if (MetadataAsValue *MAV =
5032 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5033 Metadata *MD = MAV->getMetadata();
5034 return dyn_cast_if_present<MDNode>(MD);
5035 }
5036 }
5037 return nullptr;
5038}
5039
5040/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5041static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5042 if (Op < CI->arg_size())
5043 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5044 return MAV->getMetadata();
5045 return nullptr;
5046}
5047
5048/// Convert debug intrinsic calls to non-instruction debug records.
5049/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5050/// \p CI - The debug intrinsic call.
5051static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5052 DbgRecord *DR = nullptr;
5053 if (Name == "label") {
5054 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5055 } else if (Name == "assign") {
5056 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5057 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5058 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5059 unwrapMAVMetadataOp(CI, 4),
5060 /*The address is a Value ref, it will be stored as a Metadata */
5061 unwrapMAVOp(CI, 5));
5062 } else if (Name == "declare") {
5063 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5064 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5065 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5066 } else if (Name == "addr") {
5067 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5068 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5069 // Don't try to add something to the expression if it's not an expression.
5070 // Instead, allow the verifier to fail later.
5071 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5072 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5073 }
5074 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5075 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5076 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5077 } else if (Name == "value") {
5078 // An old version of dbg.value had an extra offset argument.
5079 unsigned VarOp = 1;
5080 unsigned ExprOp = 2;
5081 if (CI->arg_size() == 4) {
5082 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5083 // Nonzero offset dbg.values get dropped without a replacement.
5084 if (!Offset || !Offset->isNullValue())
5085 return;
5086 VarOp = 2;
5087 ExprOp = 3;
5088 }
5089 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5090 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5091 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5092 nullptr);
5093 }
5094 DR->setDebugLoc(CI->getDebugLoc());
5095 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5096 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5097}
5098
5099static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5100 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5101 if (!Offset)
5102 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5103 int64_t OffsetVal = Offset->getSExtValue();
5104 return Builder.CreateIntrinsic(OffsetVal >= 0
5105 ? Intrinsic::vector_splice_left
5106 : Intrinsic::vector_splice_right,
5107 CI->getType(),
5108 {CI->getArgOperand(0), CI->getArgOperand(1),
5109 Builder.getInt32(std::abs(OffsetVal))});
5110}
5111
5112static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5113 Function *F, IRBuilder<> &Builder) {
5114 if (Name.starts_with("to.fp16")) {
5115 Value *Cast =
5116 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5117 return Builder.CreateBitCast(Cast, CI->getType());
5118 }
5119
5120 if (Name.starts_with("from.fp16")) {
5121 Value *Cast =
5122 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5123 return Builder.CreateFPExt(Cast, CI->getType());
5124 }
5125
5126 return nullptr;
5127}
5128
5129static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5130 IRBuilder<> &Builder) {
5131 Intrinsic::ID IID = NewFn->getIntrinsicID();
5132
5133 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5134 if (Defaults.empty())
5135 return false;
5136
5137 unsigned OldArgCount = CI->arg_size();
5138 unsigned NewArgCount = NewFn->arg_size();
5139
5140 // If the caller already supplied all arguments (or more), nothing to do.
5141 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5142 if (OldArgCount >= NewArgCount)
5143 return false;
5144
5145 // Start with the existing arguments from the old call.
5146 SmallVector<Value *, 8> NewArgs(CI->args());
5147
5148 // Defaults are a contiguous trailing block, so checking the first missing
5149 // argument is enough.
5150 if (OldArgCount < FirstDefault)
5151 return false;
5152
5153 // Fill in each missing trailing argument from the table.
5154 FunctionType *NewFT = NewFn->getFunctionType();
5155 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5156 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5157 "missing argument outside the default range");
5158 Type *ParamTy = NewFT->getParamType(Idx);
5159
5160 // Only integer types are supported (i1, i8, i16, i32, i64).
5161 if (!ParamTy->isIntegerTy())
5162 return false;
5163 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5164 }
5165
5166 // Preserve operand bundles by creating the call with them.
5167 SmallVector<OperandBundleDef, 1> OpBundles;
5168 CI->getOperandBundlesAsDefs(OpBundles);
5169 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5170
5171 NewCall->takeName(CI);
5172 NewCall->setCallingConv(CI->getCallingConv());
5173 NewCall->copyMetadata(*CI);
5174 if (auto *OldCI = dyn_cast<CallInst>(CI))
5175 NewCall->setTailCallKind(OldCI->getTailCallKind());
5176
5177 CI->replaceAllUsesWith(NewCall);
5178 CI->eraseFromParent();
5179 return true;
5180}
5181
5182/// Upgrade a call to an old intrinsic. All argument and return casting must be
5183/// provided to seamlessly integrate with existing context.
5184void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5185 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5186 // checks the callee's function type matches. It's likely we need to handle
5187 // type changes here.
5188 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5189 if (!F)
5190 return;
5191
5192 LLVMContext &C = CI->getContext();
5193 IRBuilder<> Builder(C);
5194 if (isa<FPMathOperator>(CI))
5195 Builder.setFastMathFlags(CI->getFastMathFlags());
5196 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5197
5198 if (!NewFn) {
5199 // Get the Function's name.
5200 StringRef Name = F->getName();
5201 if (!Name.consume_front("llvm."))
5202 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5203
5204 bool IsX86 = Name.consume_front("x86.");
5205 bool IsNVVM = Name.consume_front("nvvm.");
5206 bool IsAArch64 = Name.consume_front("aarch64.");
5207 bool IsARM = Name.consume_front("arm.");
5208 bool IsAMDGCN = Name.consume_front("amdgcn.");
5209 bool IsDbg = Name.consume_front("dbg.");
5210 bool IsOldSplice =
5211 (Name.consume_front("experimental.vector.splice") ||
5212 Name.consume_front("vector.splice")) &&
5213 !(Name.starts_with(".left") || Name.starts_with(".right"));
5214 Value *Rep = nullptr;
5215
5216 if (!IsX86 && Name == "stackprotectorcheck") {
5217 Rep = nullptr;
5218 } else if (IsNVVM) {
5219 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5220 } else if (IsX86) {
5221 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5222 } else if (IsAArch64) {
5223 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5224 } else if (IsARM) {
5225 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5226 } else if (IsAMDGCN) {
5227 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5228 } else if (IsDbg) {
5229 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5230 } else if (IsOldSplice) {
5231 Rep = upgradeVectorSplice(CI, Builder);
5232 } else if (Name.consume_front("convert.")) {
5233 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5234 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5235 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5236 Rep = nullptr;
5237 } else {
5238 llvm_unreachable("Unknown function for CallBase upgrade.");
5239 }
5240
5241 if (Rep)
5242 CI->replaceAllUsesWith(Rep);
5243 CI->eraseFromParent();
5244 return;
5245 }
5246
5247 const auto &DefaultCase = [&]() -> void {
5248 if (F == NewFn)
5249 return;
5250
5251 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5252 // Handle generic mangling change.
5253 assert(
5254 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5255 "Unknown function for CallBase upgrade and isn't just a name change");
5256 CI->setCalledFunction(NewFn);
5257 return;
5258 }
5259
5260 // This must be an upgrade from a named to a literal struct.
5261 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5262 assert(OldST != NewFn->getReturnType() &&
5263 "Return type must have changed");
5264 assert(OldST->getNumElements() ==
5265 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5266 "Must have same number of elements");
5267
5268 SmallVector<Value *> Args(CI->args());
5269 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5270 NewCI->setAttributes(CI->getAttributes());
5271 Value *Res = PoisonValue::get(OldST);
5272 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5273 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5274 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5275 }
5276 CI->replaceAllUsesWith(Res);
5277 CI->eraseFromParent();
5278 return;
5279 }
5280
5281 // We're probably about to produce something invalid. Let the verifier catch
5282 // it instead of dying here.
5283 CI->setCalledOperand(
5284 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5285 return;
5286 };
5287 CallInst *NewCall = nullptr;
5288 switch (NewFn->getIntrinsicID()) {
5289 default: {
5290 // Last resort: try the data-driven default-arg upgrade.
5291 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5292 // in its .td definition, without needing a dedicated case.
5293 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5294 return;
5295 DefaultCase();
5296 return;
5297 }
5298 case Intrinsic::arm_neon_vst1:
5299 case Intrinsic::arm_neon_vst2:
5300 case Intrinsic::arm_neon_vst3:
5301 case Intrinsic::arm_neon_vst4:
5302 case Intrinsic::arm_neon_vst2lane:
5303 case Intrinsic::arm_neon_vst3lane:
5304 case Intrinsic::arm_neon_vst4lane: {
5305 SmallVector<Value *, 4> Args(CI->args());
5306 NewCall = Builder.CreateCall(NewFn, Args);
5307 break;
5308 }
5309 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5310 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5311 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5312 LLVMContext &Ctx = F->getParent()->getContext();
5313 SmallVector<Value *, 4> Args(CI->args());
5314 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5315 cast<ConstantInt>(Args[3])->getZExtValue());
5316 NewCall = Builder.CreateCall(NewFn, Args);
5317 break;
5318 }
5319 case Intrinsic::aarch64_sve_ld3_sret:
5320 case Intrinsic::aarch64_sve_ld4_sret:
5321 case Intrinsic::aarch64_sve_ld2_sret: {
5322 // Is this a trivial remangle of the name to support ptr address spaces?
5323 if (isa<StructType>(F->getReturnType())) {
5324 DefaultCase();
5325 return;
5326 }
5327
5328 StringRef Name = F->getName();
5329 Name = Name.substr(5);
5330 unsigned N = StringSwitch<unsigned>(Name)
5331 .StartsWith("aarch64.sve.ld2", 2)
5332 .StartsWith("aarch64.sve.ld3", 3)
5333 .StartsWith("aarch64.sve.ld4", 4)
5334 .Default(0);
5335 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5336 unsigned MinElts = RetTy->getMinNumElements() / N;
5337 SmallVector<Value *, 2> Args(CI->args());
5338 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5339 Value *Ret = llvm::PoisonValue::get(RetTy);
5340 for (unsigned I = 0; I < N; I++) {
5341 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5342 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5343 }
5344 NewCall = dyn_cast<CallInst>(Ret);
5345 break;
5346 }
5347
5348 case Intrinsic::coro_end_async:
5349 case Intrinsic::coro_end: {
5350 SmallVector<Value *, 3> Args(CI->args());
5351 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5352 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5353 NewCall = Builder.CreateCall(NewFn, Args);
5354
5355 if (!CI->getType()->isVoidTy()) {
5356 if (!CI->use_empty()) {
5357 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5358 CI->getModule(), Intrinsic::coro_is_in_ramp);
5359 Value *InRamp = Builder.CreateCall(IsInRamp);
5360 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5361 }
5362 CI->eraseFromParent();
5363 return;
5364 }
5365
5366 break;
5367 }
5368
5369 case Intrinsic::vector_extract: {
5370 StringRef Name = F->getName();
5371 Name = Name.substr(5); // Strip llvm
5372 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5373 DefaultCase();
5374 return;
5375 }
5376 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5377 unsigned MinElts = RetTy->getMinNumElements();
5378 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5379 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5380 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5381 break;
5382 }
5383
5384 case Intrinsic::vector_insert: {
5385 StringRef Name = F->getName();
5386 Name = Name.substr(5);
5387 if (!Name.starts_with("aarch64.sve.tuple")) {
5388 DefaultCase();
5389 return;
5390 }
5391 if (Name.starts_with("aarch64.sve.tuple.set")) {
5392 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5393 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5394 Value *NewIdx =
5395 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5396 NewCall = Builder.CreateCall(
5397 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5398 break;
5399 }
5400 if (Name.starts_with("aarch64.sve.tuple.create")) {
5401 unsigned N = StringSwitch<unsigned>(Name)
5402 .StartsWith("aarch64.sve.tuple.create2", 2)
5403 .StartsWith("aarch64.sve.tuple.create3", 3)
5404 .StartsWith("aarch64.sve.tuple.create4", 4)
5405 .Default(0);
5406 assert(N > 1 && "Create is expected to be between 2-4");
5407 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5408 Value *Ret = llvm::PoisonValue::get(RetTy);
5409 unsigned MinElts = RetTy->getMinNumElements() / N;
5410 for (unsigned I = 0; I < N; I++) {
5411 Value *V = CI->getArgOperand(I);
5412 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5413 }
5414 NewCall = dyn_cast<CallInst>(Ret);
5415 }
5416 break;
5417 }
5418
5419 case Intrinsic::arm_neon_bfdot:
5420 case Intrinsic::arm_neon_bfmmla:
5421 case Intrinsic::arm_neon_bfmlalb:
5422 case Intrinsic::arm_neon_bfmlalt:
5423 case Intrinsic::aarch64_neon_bfdot:
5424 case Intrinsic::aarch64_neon_bfmmla:
5425 case Intrinsic::aarch64_neon_bfmlalb:
5426 case Intrinsic::aarch64_neon_bfmlalt: {
5427 SmallVector<Value *, 3> Args;
5428 assert(CI->arg_size() == 3 &&
5429 "Mismatch between function args and call args");
5430 size_t OperandWidth =
5431 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5432 assert((OperandWidth == 64 || OperandWidth == 128) &&
5433 "Unexpected operand width");
5434 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5435 auto Iter = CI->args().begin();
5436 Args.push_back(*Iter++);
5437 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5438 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5439 NewCall = Builder.CreateCall(NewFn, Args);
5440 break;
5441 }
5442
5443 case Intrinsic::bitreverse:
5444 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5445 break;
5446
5447 case Intrinsic::ctlz:
5448 case Intrinsic::cttz: {
5449 if (CI->arg_size() != 1) {
5450 DefaultCase();
5451 return;
5452 }
5453
5454 NewCall =
5455 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5456 break;
5457 }
5458
5459 case Intrinsic::objectsize: {
5460 Value *NullIsUnknownSize =
5461 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5462 Value *Dynamic =
5463 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5464 NewCall = Builder.CreateCall(
5465 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5466 break;
5467 }
5468
5469 case Intrinsic::ctpop:
5470 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5471 break;
5472 case Intrinsic::dbg_value: {
5473 StringRef Name = F->getName();
5474 Name = Name.substr(5); // Strip llvm.
5475 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5476 if (Name.starts_with("dbg.addr")) {
5477 DIExpression *Expr = cast<DIExpression>(
5478 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5479 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5480 NewCall =
5481 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5482 MetadataAsValue::get(C, Expr)});
5483 break;
5484 }
5485
5486 // Upgrade from the old version that had an extra offset argument.
5487 assert(CI->arg_size() == 4);
5488 // Drop nonzero offsets instead of attempting to upgrade them.
5489 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5490 if (Offset->isNullValue()) {
5491 NewCall = Builder.CreateCall(
5492 NewFn,
5493 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5494 break;
5495 }
5496 CI->eraseFromParent();
5497 return;
5498 }
5499
5500 case Intrinsic::ptr_annotation:
5501 // Upgrade from versions that lacked the annotation attribute argument.
5502 if (CI->arg_size() != 4) {
5503 DefaultCase();
5504 return;
5505 }
5506
5507 // Create a new call with an added null annotation attribute argument.
5508 NewCall = Builder.CreateCall(
5509 NewFn,
5510 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5511 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5512 NewCall->takeName(CI);
5513 CI->replaceAllUsesWith(NewCall);
5514 CI->eraseFromParent();
5515 return;
5516
5517 case Intrinsic::var_annotation:
5518 // Upgrade from versions that lacked the annotation attribute argument.
5519 if (CI->arg_size() != 4) {
5520 DefaultCase();
5521 return;
5522 }
5523 // Create a new call with an added null annotation attribute argument.
5524 NewCall = Builder.CreateCall(
5525 NewFn,
5526 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5527 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5528 NewCall->takeName(CI);
5529 CI->replaceAllUsesWith(NewCall);
5530 CI->eraseFromParent();
5531 return;
5532
5533 case Intrinsic::riscv_aes32dsi:
5534 case Intrinsic::riscv_aes32dsmi:
5535 case Intrinsic::riscv_aes32esi:
5536 case Intrinsic::riscv_aes32esmi:
5537 case Intrinsic::riscv_sm4ks:
5538 case Intrinsic::riscv_sm4ed: {
5539 // The last argument to these intrinsics used to be i8 and changed to i32.
5540 // The type overload for sm4ks and sm4ed was removed.
5541 Value *Arg2 = CI->getArgOperand(2);
5542 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5543 return;
5544
5545 Value *Arg0 = CI->getArgOperand(0);
5546 Value *Arg1 = CI->getArgOperand(1);
5547 if (CI->getType()->isIntegerTy(64)) {
5548 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5549 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5550 }
5551
5552 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5553 cast<ConstantInt>(Arg2)->getZExtValue());
5554
5555 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5556 Value *Res = NewCall;
5557 if (Res->getType() != CI->getType())
5558 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5559 NewCall->takeName(CI);
5560 CI->replaceAllUsesWith(Res);
5561 CI->eraseFromParent();
5562 return;
5563 }
5564 case Intrinsic::nvvm_mapa_shared_cluster: {
5565 // Create a new call with the correct address space.
5566 NewCall =
5567 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5568 Value *Res = NewCall;
5569 Res = Builder.CreateAddrSpaceCast(
5570 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5571 NewCall->takeName(CI);
5572 CI->replaceAllUsesWith(Res);
5573 CI->eraseFromParent();
5574 return;
5575 }
5576 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5577 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5578 // Create a new call with the correct address space.
5579 SmallVector<Value *, 4> Args(CI->args());
5580 Args[0] = Builder.CreateAddrSpaceCast(
5581 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5582
5583 NewCall = Builder.CreateCall(NewFn, Args);
5584 NewCall->takeName(CI);
5585 CI->replaceAllUsesWith(NewCall);
5586 CI->eraseFromParent();
5587 return;
5588 }
5589 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5590 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5591 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5592 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5593 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5594 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5595 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5596 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5597 SmallVector<Value *, 16> Args(CI->args());
5598
5599 // Create AddrSpaceCast to shared_cluster if needed.
5600 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5601 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5602 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5603 Args[0] = Builder.CreateAddrSpaceCast(
5604 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5605
5606 // Attach the flag argument for cta_group, with a
5607 // default value of 0. This handles case (2) in
5608 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5609 size_t NumArgs = CI->arg_size();
5610 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5611 if (!FlagArg->getType()->isIntegerTy(1))
5612 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5613
5614 NewCall = Builder.CreateCall(NewFn, Args);
5615 NewCall->takeName(CI);
5616 CI->replaceAllUsesWith(NewCall);
5617 CI->eraseFromParent();
5618 return;
5619 }
5620 case Intrinsic::riscv_sha256sig0:
5621 case Intrinsic::riscv_sha256sig1:
5622 case Intrinsic::riscv_sha256sum0:
5623 case Intrinsic::riscv_sha256sum1:
5624 case Intrinsic::riscv_sm3p0:
5625 case Intrinsic::riscv_sm3p1: {
5626 // The last argument to these intrinsics used to be i8 and changed to i32.
5627 // The type overload for sm4ks and sm4ed was removed.
5628 if (!CI->getType()->isIntegerTy(64))
5629 return;
5630
5631 Value *Arg =
5632 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
5633
5634 NewCall = Builder.CreateCall(NewFn, Arg);
5635 Value *Res =
5636 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5637 NewCall->takeName(CI);
5638 CI->replaceAllUsesWith(Res);
5639 CI->eraseFromParent();
5640 return;
5641 }
5642
5643 case Intrinsic::x86_xop_vfrcz_ss:
5644 case Intrinsic::x86_xop_vfrcz_sd:
5645 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
5646 break;
5647
5648 case Intrinsic::x86_xop_vpermil2pd:
5649 case Intrinsic::x86_xop_vpermil2ps:
5650 case Intrinsic::x86_xop_vpermil2pd_256:
5651 case Intrinsic::x86_xop_vpermil2ps_256: {
5652 SmallVector<Value *, 4> Args(CI->args());
5653 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
5654 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
5655 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
5656 NewCall = Builder.CreateCall(NewFn, Args);
5657 break;
5658 }
5659
5660 case Intrinsic::x86_sse41_ptestc:
5661 case Intrinsic::x86_sse41_ptestz:
5662 case Intrinsic::x86_sse41_ptestnzc: {
5663 // The arguments for these intrinsics used to be v4f32, and changed
5664 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
5665 // So, the only thing required is a bitcast for both arguments.
5666 // First, check the arguments have the old type.
5667 Value *Arg0 = CI->getArgOperand(0);
5668 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
5669 return;
5670
5671 // Old intrinsic, add bitcasts
5672 Value *Arg1 = CI->getArgOperand(1);
5673
5674 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
5675
5676 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
5677 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
5678
5679 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
5680 break;
5681 }
5682
5683 case Intrinsic::x86_rdtscp: {
5684 // This used to take 1 arguments. If we have no arguments, it is already
5685 // upgraded.
5686 if (CI->getNumOperands() == 0)
5687 return;
5688
5689 NewCall = Builder.CreateCall(NewFn);
5690 // Extract the second result and store it.
5691 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5692 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
5693 // Replace the original call result with the first result of the new call.
5694 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
5695
5696 NewCall->takeName(CI);
5697 CI->replaceAllUsesWith(TSC);
5698 CI->eraseFromParent();
5699 return;
5700 }
5701
5702 case Intrinsic::x86_sse41_insertps:
5703 case Intrinsic::x86_sse41_dppd:
5704 case Intrinsic::x86_sse41_dpps:
5705 case Intrinsic::x86_sse41_mpsadbw:
5706 case Intrinsic::x86_avx_dp_ps_256:
5707 case Intrinsic::x86_avx2_mpsadbw: {
5708 // Need to truncate the last argument from i32 to i8 -- this argument models
5709 // an inherently 8-bit immediate operand to these x86 instructions.
5710 SmallVector<Value *, 4> Args(CI->args());
5711
5712 // Replace the last argument with a trunc.
5713 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
5714 NewCall = Builder.CreateCall(NewFn, Args);
5715 break;
5716 }
5717
5718 case Intrinsic::x86_avx512_mask_cmp_pd_128:
5719 case Intrinsic::x86_avx512_mask_cmp_pd_256:
5720 case Intrinsic::x86_avx512_mask_cmp_pd_512:
5721 case Intrinsic::x86_avx512_mask_cmp_ps_128:
5722 case Intrinsic::x86_avx512_mask_cmp_ps_256:
5723 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
5724 SmallVector<Value *, 4> Args(CI->args());
5725 unsigned NumElts =
5726 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
5727 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
5728
5729 NewCall = Builder.CreateCall(NewFn, Args);
5730 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
5731
5732 NewCall->takeName(CI);
5733 CI->replaceAllUsesWith(Res);
5734 CI->eraseFromParent();
5735 return;
5736 }
5737
5738 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
5739 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
5740 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
5741 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
5742 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
5743 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
5744 SmallVector<Value *, 4> Args(CI->args());
5745 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
5746 if (NewFn->getIntrinsicID() ==
5747 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
5748 Args[1] = Builder.CreateBitCast(
5749 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5750
5751 NewCall = Builder.CreateCall(NewFn, Args);
5752 Value *Res = Builder.CreateBitCast(
5753 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
5754
5755 NewCall->takeName(CI);
5756 CI->replaceAllUsesWith(Res);
5757 CI->eraseFromParent();
5758 return;
5759 }
5760 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
5761 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
5762 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
5763 SmallVector<Value *, 4> Args(CI->args());
5764 unsigned NumElts =
5765 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
5766 Args[1] = Builder.CreateBitCast(
5767 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5768 Args[2] = Builder.CreateBitCast(
5769 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5770
5771 NewCall = Builder.CreateCall(NewFn, Args);
5772 break;
5773 }
5774
5775 case Intrinsic::thread_pointer: {
5776 NewCall = Builder.CreateCall(NewFn, {});
5777 break;
5778 }
5779
5780 case Intrinsic::memcpy:
5781 case Intrinsic::memmove:
5782 case Intrinsic::memset: {
5783 // We have to make sure that the call signature is what we're expecting.
5784 // We only want to change the old signatures by removing the alignment arg:
5785 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
5786 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
5787 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
5788 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
5789 // Note: i8*'s in the above can be any pointer type
5790 if (CI->arg_size() != 5) {
5791 DefaultCase();
5792 return;
5793 }
5794 // Remove alignment argument (3), and add alignment attributes to the
5795 // dest/src pointers.
5796 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
5797 CI->getArgOperand(2), CI->getArgOperand(4)};
5798 NewCall = Builder.CreateCall(NewFn, Args);
5799 AttributeList OldAttrs = CI->getAttributes();
5800 AttributeList NewAttrs = AttributeList::get(
5801 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
5802 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
5803 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
5804 NewCall->setAttributes(NewAttrs);
5805 auto *MemCI = cast<MemIntrinsic>(NewCall);
5806 // All mem intrinsics support dest alignment.
5807 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
5808 MemCI->setDestAlignment(Align->getMaybeAlignValue());
5809 // Memcpy/Memmove also support source alignment.
5810 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
5811 MTI->setSourceAlignment(Align->getMaybeAlignValue());
5812 break;
5813 }
5814
5815 case Intrinsic::masked_load:
5816 case Intrinsic::masked_gather:
5817 case Intrinsic::masked_store:
5818 case Intrinsic::masked_scatter: {
5819 if (CI->arg_size() != 4) {
5820 DefaultCase();
5821 return;
5822 }
5823
5824 auto GetMaybeAlign = [](Value *Op) {
5825 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
5826 uint64_t Val = CI->getZExtValue();
5827 if (Val == 0)
5828 return MaybeAlign();
5829 if (isPowerOf2_64(Val))
5830 return MaybeAlign(Val);
5831 }
5832 reportFatalUsageError("Invalid alignment argument");
5833 };
5834 auto GetAlign = [&](Value *Op) {
5835 MaybeAlign Align = GetMaybeAlign(Op);
5836 if (Align)
5837 return *Align;
5838 reportFatalUsageError("Invalid zero alignment argument");
5839 };
5840
5841 const DataLayout &DL = CI->getDataLayout();
5842 switch (NewFn->getIntrinsicID()) {
5843 case Intrinsic::masked_load:
5844 NewCall = Builder.CreateMaskedLoad(
5845 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
5846 CI->getArgOperand(2), CI->getArgOperand(3));
5847 break;
5848 case Intrinsic::masked_gather:
5849 NewCall = Builder.CreateMaskedGather(
5850 CI->getType(), CI->getArgOperand(0),
5851 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
5852 CI->getType()->getScalarType()),
5853 CI->getArgOperand(2), CI->getArgOperand(3));
5854 break;
5855 case Intrinsic::masked_store:
5856 NewCall = Builder.CreateMaskedStore(
5857 CI->getArgOperand(0), CI->getArgOperand(1),
5858 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
5859 break;
5860 case Intrinsic::masked_scatter:
5861 NewCall = Builder.CreateMaskedScatter(
5862 CI->getArgOperand(0), CI->getArgOperand(1),
5863 DL.getValueOrABITypeAlignment(
5864 GetMaybeAlign(CI->getArgOperand(2)),
5865 CI->getArgOperand(0)->getType()->getScalarType()),
5866 CI->getArgOperand(3));
5867 break;
5868 default:
5869 llvm_unreachable("Unexpected intrinsic ID");
5870 }
5871 // Previous metadata is still valid.
5872 NewCall->copyMetadata(*CI);
5873 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5874 break;
5875 }
5876
5877 case Intrinsic::lifetime_start:
5878 case Intrinsic::lifetime_end: {
5879 if (CI->arg_size() != 2) {
5880 DefaultCase();
5881 return;
5882 }
5883
5884 Value *Ptr = CI->getArgOperand(1);
5885 // Try to strip pointer casts, such that the lifetime works on an alloca.
5886 Ptr = Ptr->stripPointerCasts();
5887 if (isa<AllocaInst>(Ptr)) {
5888 // Don't use NewFn, as we might have looked through an addrspacecast.
5889 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
5890 NewCall = Builder.CreateLifetimeStart(Ptr);
5891 else
5892 NewCall = Builder.CreateLifetimeEnd(Ptr);
5893 break;
5894 }
5895
5896 // Otherwise remove the lifetime marker.
5897 CI->eraseFromParent();
5898 return;
5899 }
5900
5901 case Intrinsic::x86_avx512_vpdpbusd_128:
5902 case Intrinsic::x86_avx512_vpdpbusd_256:
5903 case Intrinsic::x86_avx512_vpdpbusd_512:
5904 case Intrinsic::x86_avx512_vpdpbusds_128:
5905 case Intrinsic::x86_avx512_vpdpbusds_256:
5906 case Intrinsic::x86_avx512_vpdpbusds_512:
5907 case Intrinsic::x86_avx2_vpdpbssd_128:
5908 case Intrinsic::x86_avx2_vpdpbssd_256:
5909 case Intrinsic::x86_avx10_vpdpbssd_512:
5910 case Intrinsic::x86_avx2_vpdpbssds_128:
5911 case Intrinsic::x86_avx2_vpdpbssds_256:
5912 case Intrinsic::x86_avx10_vpdpbssds_512:
5913 case Intrinsic::x86_avx2_vpdpbsud_128:
5914 case Intrinsic::x86_avx2_vpdpbsud_256:
5915 case Intrinsic::x86_avx10_vpdpbsud_512:
5916 case Intrinsic::x86_avx2_vpdpbsuds_128:
5917 case Intrinsic::x86_avx2_vpdpbsuds_256:
5918 case Intrinsic::x86_avx10_vpdpbsuds_512:
5919 case Intrinsic::x86_avx2_vpdpbuud_128:
5920 case Intrinsic::x86_avx2_vpdpbuud_256:
5921 case Intrinsic::x86_avx10_vpdpbuud_512:
5922 case Intrinsic::x86_avx2_vpdpbuuds_128:
5923 case Intrinsic::x86_avx2_vpdpbuuds_256:
5924 case Intrinsic::x86_avx10_vpdpbuuds_512: {
5925 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
5926 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5927 CI->getArgOperand(2)};
5928 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
5929 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5930 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5931
5932 NewCall = Builder.CreateCall(NewFn, Args);
5933 break;
5934 }
5935 case Intrinsic::x86_avx512_vpdpwssd_128:
5936 case Intrinsic::x86_avx512_vpdpwssd_256:
5937 case Intrinsic::x86_avx512_vpdpwssd_512:
5938 case Intrinsic::x86_avx512_vpdpwssds_128:
5939 case Intrinsic::x86_avx512_vpdpwssds_256:
5940 case Intrinsic::x86_avx512_vpdpwssds_512:
5941 case Intrinsic::x86_avx2_vpdpwsud_128:
5942 case Intrinsic::x86_avx2_vpdpwsud_256:
5943 case Intrinsic::x86_avx10_vpdpwsud_512:
5944 case Intrinsic::x86_avx2_vpdpwsuds_128:
5945 case Intrinsic::x86_avx2_vpdpwsuds_256:
5946 case Intrinsic::x86_avx10_vpdpwsuds_512:
5947 case Intrinsic::x86_avx2_vpdpwusd_128:
5948 case Intrinsic::x86_avx2_vpdpwusd_256:
5949 case Intrinsic::x86_avx10_vpdpwusd_512:
5950 case Intrinsic::x86_avx2_vpdpwusds_128:
5951 case Intrinsic::x86_avx2_vpdpwusds_256:
5952 case Intrinsic::x86_avx10_vpdpwusds_512:
5953 case Intrinsic::x86_avx2_vpdpwuud_128:
5954 case Intrinsic::x86_avx2_vpdpwuud_256:
5955 case Intrinsic::x86_avx10_vpdpwuud_512:
5956 case Intrinsic::x86_avx2_vpdpwuuds_128:
5957 case Intrinsic::x86_avx2_vpdpwuuds_256:
5958 case Intrinsic::x86_avx10_vpdpwuuds_512:
5959 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
5960 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5961 CI->getArgOperand(2)};
5962 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
5963 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5964 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5965
5966 NewCall = Builder.CreateCall(NewFn, Args);
5967 break;
5968 }
5969 assert(NewCall && "Should have either set this variable or returned through "
5970 "the default case");
5971 NewCall->takeName(CI);
5972 CI->replaceAllUsesWith(NewCall);
5973 CI->eraseFromParent();
5974}
5975
5976void llvm::UpgradeCallsToIntrinsic(Function *F) {
5977 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
5978
5979 // Check if this function should be upgraded and get the replacement function
5980 // if there is one.
5981 Function *NewFn;
5982 if (UpgradeIntrinsicFunction(F, NewFn)) {
5983 // Replace all users of the old function with the new function or new
5984 // instructions. This is not a range loop because the call is deleted.
5985 for (User *U : make_early_inc_range(F->users()))
5986 if (CallBase *CB = dyn_cast<CallBase>(U))
5987 UpgradeIntrinsicCall(CB, NewFn);
5988
5989 // Remove old function, no longer used, from the module.
5990 if (F != NewFn)
5991 F->eraseFromParent();
5992 }
5993}
5994
5995MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
5996 const unsigned NumOperands = MD.getNumOperands();
5997 if (NumOperands == 0)
5998 return &MD; // Invalid, punt to a verifier error.
5999
6000 // Check if the tag uses struct-path aware TBAA format.
6001 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6002 return &MD;
6003
6004 auto &Context = MD.getContext();
6005 if (NumOperands == 3) {
6006 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6007 MDNode *ScalarType = MDNode::get(Context, Elts);
6008 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6009 Metadata *Elts2[] = {ScalarType, ScalarType,
6010 ConstantAsMetadata::get(
6011 Constant::getNullValue(Type::getInt64Ty(Context))),
6012 MD.getOperand(2)};
6013 return MDNode::get(Context, Elts2);
6014 }
6015 // Create a MDNode <MD, MD, offset 0>
6016 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6017 Type::getInt64Ty(Context)))};
6018 return MDNode::get(Context, Elts);
6019}
6020
6021Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6022 Instruction *&Temp) {
6023 if (Opc != Instruction::BitCast)
6024 return nullptr;
6025
6026 Temp = nullptr;
6027 Type *SrcTy = V->getType();
6028 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6029 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6030 LLVMContext &Context = V->getContext();
6031
6032 // We have no information about target data layout, so we assume that
6033 // the maximum pointer size is 64bit.
6034 Type *MidTy = Type::getInt64Ty(Context);
6035 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6036
6037 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6038 }
6039
6040 return nullptr;
6041}
6042
6043Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6044 if (Opc != Instruction::BitCast)
6045 return nullptr;
6046
6047 Type *SrcTy = C->getType();
6048 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6049 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6050 LLVMContext &Context = C->getContext();
6051
6052 // We have no information about target data layout, so we assume that
6053 // the maximum pointer size is 64bit.
6054 Type *MidTy = Type::getInt64Ty(Context);
6055
6056 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6057 DestTy);
6058 }
6059
6060 return nullptr;
6061}
6062
6063static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6064 if (Flag.getNumOperands() < 3)
6065 return std::nullopt;
6066 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6067 return Name->getString();
6068 return std::nullopt;
6069}
6070
6071/// Check the debug info version number, if it is out-dated, drop the debug
6072/// info. Return true if module is modified.
6073bool llvm::UpgradeDebugInfo(Module &M) {
6074 if (DisableAutoUpgradeDebugInfo)
6075 return false;
6076
6077 llvm::TimeTraceScope timeScope("Upgrade debug info");
6078 // We need to get metadata before the module is verified (i.e., getModuleFlag
6079 // makes assumptions that we haven't verified yet). Carefully extract the flag
6080 // from the metadata.
6081 unsigned Version = 0;
6082 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6083 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6084 if (auto Name = getModuleFlagNameSafely(*Flag))
6085 return *Name == "Debug Info Version";
6086 return false;
6087 });
6088 if (OpIt != ModFlags->op_end()) {
6089 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6090 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6091 Version = CI->getZExtValue();
6092 }
6093 }
6094
6095 if (Version == DEBUG_METADATA_VERSION) {
6096 bool BrokenDebugInfo = false;
6097 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6098 report_fatal_error("Broken module found, compilation aborted!");
6099 if (!BrokenDebugInfo)
6100 // Everything is ok.
6101 return false;
6102 else {
6103 // Diagnose malformed debug info.
6104 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6105 M.getContext().diagnose(Diag);
6106 }
6107 }
6108 bool Modified = StripDebugInfo(M);
6109 if (Modified && Version != DEBUG_METADATA_VERSION) {
6110 // Diagnose a version mismatch.
6111 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6112 M.getContext().diagnose(DiagVersion);
6113 }
6114 return Modified;
6115}
6116
6117static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6118 GlobalValue *GV, const Metadata *V) {
6119 Function *F = cast<Function>(GV);
6120
6121 constexpr StringLiteral DefaultValue = "1";
6122 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6123 unsigned Length = 0;
6124
6125 if (F->hasFnAttribute(Attr)) {
6126 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6127 // parse these elements placing them into Vect3
6128 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6129 for (; Length < 3 && !S.empty(); Length++) {
6130 auto [Part, Rest] = S.split(',');
6131 Vect3[Length] = Part.trim();
6132 S = Rest;
6133 }
6134 }
6135
6136 const unsigned Dim = DimC - 'x';
6137 assert(Dim < 3 && "Unexpected dim char");
6138
6139 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6140
6141 // local variable required for StringRef in Vect3 to point to.
6142 const std::string VStr = llvm::utostr(VInt);
6143 Vect3[Dim] = VStr;
6144 Length = std::max(Length, Dim + 1);
6145
6146 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6147 F->addFnAttr(Attr, NewAttr);
6148}
6149
6150static inline bool isXYZ(StringRef S) {
6151 return S == "x" || S == "y" || S == "z";
6152}
6153
6154bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6155 const Metadata *V) {
6156 if (K == "kernel") {
6157 if (!mdconst::extract<ConstantInt>(V)->isZero())
6158 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6159 return true;
6160 }
6161 if (K == "align") {
6162 // V is a bitfeild specifying two 16-bit values. The alignment value is
6163 // specfied in low 16-bits, The index is specified in the high bits. For the
6164 // index, 0 indicates the return value while higher values correspond to
6165 // each parameter (idx = param + 1).
6166 const uint64_t AlignIdxValuePair =
6167 mdconst::extract<ConstantInt>(V)->getZExtValue();
6168 const unsigned Idx = (AlignIdxValuePair >> 16);
6169 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6170 cast<Function>(GV)->addAttributeAtIndex(
6171 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6172 return true;
6173 }
6174 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6175 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6176 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6177 return true;
6178 }
6179 if (K == "minctasm") {
6180 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6181 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6182 return true;
6183 }
6184 if (K == "maxnreg") {
6185 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6186 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6187 return true;
6188 }
6189 if (K.consume_front("maxntid") && isXYZ(K)) {
6190 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6191 return true;
6192 }
6193 if (K.consume_front("reqntid") && isXYZ(K)) {
6194 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6195 return true;
6196 }
6197 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6198 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6199 return true;
6200 }
6201 if (K == "grid_constant") {
6202 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6203 for (const auto &Op : cast<MDNode>(V)->operands()) {
6204 // For some reason, the index is 1-based in the metadata. Good thing we're
6205 // able to auto-upgrade it!
6206 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6207 cast<Function>(GV)->addParamAttr(Index, Attr);
6208 }
6209 return true;
6210 }
6211
6212 return false;
6213}
6214
6215void llvm::UpgradeNVVMAnnotations(Module &M) {
6216 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6217 if (!NamedMD)
6218 return;
6219
6220 SmallVector<MDNode *, 8> NewNodes;
6221 SmallPtrSet<const MDNode *, 8> SeenNodes;
6222 for (MDNode *MD : NamedMD->operands()) {
6223 if (!SeenNodes.insert(MD).second)
6224 continue;
6225
6226 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6227 if (!GV)
6228 continue;
6229
6230 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6231
6232 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6233 // Each nvvm.annotations metadata entry will be of the following form:
6234 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6235 // start index = 1, to skip the global variable key
6236 // increment = 2, to skip the value for each property-value pairs
6237 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6238 MDString *K = cast<MDString>(MD->getOperand(j));
6239 const MDOperand &V = MD->getOperand(j + 1);
6240 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6241 if (!Upgraded)
6242 NewOperands.append({K, V});
6243 }
6244
6245 if (NewOperands.size() > 1)
6246 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6247 }
6248
6249 NamedMD->clearOperands();
6250 for (MDNode *N : NewNodes)
6251 NamedMD->addOperand(N);
6252}
6253
6254/// This checks for objc retain release marker which should be upgraded. It
6255/// returns true if module is modified.
6256static bool upgradeRetainReleaseMarker(Module &M) {
6257 bool Changed = false;
6258 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6259 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6260 if (ModRetainReleaseMarker) {
6261 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6262 if (Op) {
6263 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6264 if (ID) {
6265 SmallVector<StringRef, 4> ValueComp;
6266 ID->getString().split(ValueComp, "#");
6267 if (ValueComp.size() == 2) {
6268 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6269 ID = MDString::get(M.getContext(), NewValue);
6270 }
6271 M.addModuleFlag(Module::Error, MarkerKey, ID);
6272 M.eraseNamedMetadata(ModRetainReleaseMarker);
6273 Changed = true;
6274 }
6275 }
6276 }
6277 return Changed;
6278}
6279
6280void llvm::UpgradeARCRuntime(Module &M) {
6281 // This lambda converts normal function calls to ARC runtime functions to
6282 // intrinsic calls.
6283 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6284 llvm::Intrinsic::ID IntrinsicFunc) {
6285 Function *Fn = M.getFunction(OldFunc);
6286
6287 if (!Fn)
6288 return;
6289
6290 Function *NewFn =
6291 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6292
6293 for (User *U : make_early_inc_range(Fn->users())) {
6294 CallInst *CI = dyn_cast<CallInst>(U);
6295 if (!CI || CI->getCalledFunction() != Fn)
6296 continue;
6297
6298 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6299 FunctionType *NewFuncTy = NewFn->getFunctionType();
6300 SmallVector<Value *, 2> Args;
6301
6302 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6303 // value to the return type of the old function.
6304 if (NewFuncTy->getReturnType() != CI->getType() &&
6305 !CastInst::castIsValid(Instruction::BitCast, CI,
6306 NewFuncTy->getReturnType()))
6307 continue;
6308
6309 bool InvalidCast = false;
6310
6311 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6312 Value *Arg = CI->getArgOperand(I);
6313
6314 // Bitcast argument to the parameter type of the new function if it's
6315 // not a variadic argument.
6316 if (I < NewFuncTy->getNumParams()) {
6317 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6318 // to the parameter type of the new function.
6319 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6320 NewFuncTy->getParamType(I))) {
6321 InvalidCast = true;
6322 break;
6323 }
6324 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6325 }
6326 Args.push_back(Arg);
6327 }
6328
6329 if (InvalidCast)
6330 continue;
6331
6332 // Create a call instruction that calls the new function.
6333 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6334 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6335 NewCall->takeName(CI);
6336
6337 // Bitcast the return value back to the type of the old call.
6338 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6339
6340 if (!CI->use_empty())
6341 CI->replaceAllUsesWith(NewRetVal);
6342 CI->eraseFromParent();
6343 }
6344
6345 if (Fn->use_empty())
6346 Fn->eraseFromParent();
6347 };
6348
6349 // Unconditionally convert a call to "clang.arc.use" to a call to
6350 // "llvm.objc.clang.arc.use".
6351 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6352
6353 // Upgrade the retain release marker. If there is no need to upgrade
6354 // the marker, that means either the module is already new enough to contain
6355 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6356 if (!upgradeRetainReleaseMarker(M))
6357 return;
6358
6359 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6360 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6361 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6362 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6363 {"objc_autoreleaseReturnValue",
6364 llvm::Intrinsic::objc_autoreleaseReturnValue},
6365 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6366 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6367 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6368 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6369 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6370 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6371 {"objc_release", llvm::Intrinsic::objc_release},
6372 {"objc_retain", llvm::Intrinsic::objc_retain},
6373 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6374 {"objc_retainAutoreleaseReturnValue",
6375 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6376 {"objc_retainAutoreleasedReturnValue",
6377 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6378 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6379 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6380 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6381 {"objc_unsafeClaimAutoreleasedReturnValue",
6382 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6383 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6384 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6385 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6386 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6387 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6388 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6389 {"objc_arc_annotation_topdown_bbstart",
6390 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6391 {"objc_arc_annotation_topdown_bbend",
6392 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6393 {"objc_arc_annotation_bottomup_bbstart",
6394 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6395 {"objc_arc_annotation_bottomup_bbend",
6396 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6397
6398 for (auto &I : RuntimeFuncs)
6399 UpgradeToIntrinsic(I.first, I.second);
6400}
6401
6402// Upgrade the way signing of pointers to init/fini functions is described.
6403//
6404// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6405// constants, if signing was requested. After the upgrade, these arrays contain
6406// plain function pointers and the desired signing schema is described via a
6407// pair of module flags.
6408//
6409// Note that the upgrade is only performed if all elements of *both* arrays
6410// agree on a common signing schema.
6411static bool upgradePtrauthInitFiniArrays(Module &M) {
6412 // As we cannot always decide whether the particular module should have
6413 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6414 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6415 // returns as soon as it spots any non-signed init/fini pointer: either we
6416 // should request non-signed pointers (safe to omit both flags) or there is
6417 // no common schema (and thus we do not modify anything).
6418 //
6419 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6420 // or whether we should request address diversity in addition to the basic
6421 // constant diversity. There is no value representing "decided not to sign"
6422 // for the reasons explained above.
6423 std::optional<bool> UseAddressDisc;
6424
6425 // Do not attempt upgrading if the new module flags already exist.
6426 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6427 for (const MDNode *Flag : ModFlags->operands()) {
6428 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
6429 if (Name && (*Name == "ptrauth-init-fini" ||
6430 *Name == "ptrauth-init-fini-address-discrimination"))
6431 return false;
6432 }
6433 }
6434
6435 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6436 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6437 constexpr unsigned ExpectedAddressMarker = 1;
6438
6439 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
6440 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
6441 return nullptr; // Nothing to upgrade or unknown pattern found.
6442
6443 bool HasAddressDisc;
6444 if (!CPA->hasAddressDiscriminator())
6445 HasAddressDisc = false;
6446 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
6447 HasAddressDisc = true;
6448 else
6449 return nullptr; // Unknown pattern.
6450
6451 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6452 return nullptr; // Disagreement with the decided mode.
6453
6454 UseAddressDisc = HasAddressDisc;
6455 return CPA->getPointer();
6456 };
6457
6458 // Do not apply any changes until we know the upgrade is non-ambiguous.
6459 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6460 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6461
6462 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6463 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
6464 if (!GV || !GV->hasInitializer())
6465 continue; // Skip, but it is okay to upgrade the other variable.
6466
6467 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
6468 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6469 return false;
6470
6471 std::vector<Constant *> NewStructors;
6472 NewStructors.reserve(OldStructorsArray->getNumOperands());
6473
6474 for (Use &U : OldStructorsArray->operands()) {
6475 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
6476 if (!Structor || Structor->getNumOperands() != 3)
6477 return false;
6478
6479 Constant *Prio = Structor->getOperand(0);
6480 Constant *Func = Structor->getOperand(1);
6481 Constant *Arg = Structor->getOperand(2);
6482
6483 Func = UpgradeSinglePointer(Func);
6484 if (!Func)
6485 return false;
6486
6487 NewStructors.push_back(
6488 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
6489 }
6490
6491 Constant *NewInit =
6492 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
6493 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
6494 }
6495
6496 if (GlobalArraysToUpgrade.empty())
6497 return false;
6498 assert(UseAddressDisc.has_value());
6499
6500 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6501 GV->setInitializer(NewInit);
6502
6503 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
6504 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
6505 *UseAddressDisc);
6506
6507 return true;
6508}
6509
6510bool llvm::UpgradeModuleFlags(Module &M) {
6511 bool Changed = false;
6512 Changed |= upgradePtrauthInitFiniArrays(M);
6513
6514 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6515 if (!ModFlags)
6516 return Changed;
6517
6518 bool HasObjCFlag = false, HasClassProperties = false;
6519 bool HasSwiftVersionFlag = false;
6520 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6521 uint32_t SwiftABIVersion;
6522 auto Int8Ty = Type::getInt8Ty(M.getContext());
6523 auto Int32Ty = Type::getInt32Ty(M.getContext());
6524
6525 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6526 MDNode *Op = ModFlags->getOperand(I);
6527 if (Op->getNumOperands() != 3)
6528 continue;
6529 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6530 if (!ID)
6531 continue;
6532 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6533 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6534 Type::getInt32Ty(M.getContext()), B)),
6535 MDString::get(M.getContext(), ID->getString()),
6536 Op->getOperand(2)};
6537 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6538 Changed = true;
6539 };
6540
6541 if (ID->getString() == "Objective-C Image Info Version")
6542 HasObjCFlag = true;
6543 if (ID->getString() == "Objective-C Class Properties")
6544 HasClassProperties = true;
6545 // Upgrade PIC from Error/Max to Min.
6546 if (ID->getString() == "PIC Level") {
6547 if (auto *Behavior =
6548 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6549 uint64_t V = Behavior->getLimitedValue();
6550 if (V == Module::Error || V == Module::Max)
6551 SetBehavior(Module::Min);
6552 }
6553 }
6554 // Upgrade "PIE Level" from Error to Max.
6555 if (ID->getString() == "PIE Level")
6556 if (auto *Behavior =
6557 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
6558 if (Behavior->getLimitedValue() == Module::Error)
6559 SetBehavior(Module::Max);
6560
6561 // Upgrade branch protection and return address signing module flags. The
6562 // module flag behavior for these fields were Error and now they are Min.
6563 if (ID->getString() == "branch-target-enforcement" ||
6564 ID->getString().starts_with("sign-return-address")) {
6565 if (auto *Behavior =
6566 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6567 if (Behavior->getLimitedValue() == Module::Error) {
6568 Type *Int32Ty = Type::getInt32Ty(M.getContext());
6569 Metadata *Ops[3] = {
6570 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
6571 Op->getOperand(1), Op->getOperand(2)};
6572 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6573 Changed = true;
6574 }
6575 }
6576 }
6577
6578 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
6579 // section name so that llvm-lto will not complain about mismatching
6580 // module flags that is functionally the same.
6581 if (ID->getString() == "Objective-C Image Info Section") {
6582 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
6583 SmallVector<StringRef, 4> ValueComp;
6584 Value->getString().split(ValueComp, " ");
6585 if (ValueComp.size() != 1) {
6586 std::string NewValue;
6587 for (auto &S : ValueComp)
6588 NewValue += S.str();
6589 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
6590 MDString::get(M.getContext(), NewValue)};
6591 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6592 Changed = true;
6593 }
6594 }
6595 }
6596
6597 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
6598 // If the higher bits are set, it adds new module flag for swift info.
6599 if (ID->getString() == "Objective-C Garbage Collection") {
6600 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
6601 if (Md) {
6602 assert(Md->getValue() && "Expected non-empty metadata");
6603 auto Type = Md->getValue()->getType();
6604 if (Type == Int8Ty)
6605 continue;
6606 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
6607 if ((Val & 0xff) != Val) {
6608 HasSwiftVersionFlag = true;
6609 SwiftABIVersion = (Val & 0xff00) >> 8;
6610 SwiftMajorVersion = (Val & 0xff000000) >> 24;
6611 SwiftMinorVersion = (Val & 0xff0000) >> 16;
6612 }
6613 Metadata *Ops[3] = {
6614 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
6615 Op->getOperand(1),
6616 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
6617 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6618 Changed = true;
6619 }
6620 }
6621
6622 if (ID->getString() == "amdgpu_code_object_version") {
6623 Metadata *Ops[3] = {
6624 Op->getOperand(0),
6625 MDString::get(M.getContext(), "amdhsa_code_object_version"),
6626 Op->getOperand(2)};
6627 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6628 Changed = true;
6629 }
6630
6631 // clang/PowerPC used to use "float-abi" to describe the long double format;
6632 // it has been renamed to "long-double-type", with its values changed to the
6633 // corresponding IR floating-point type names.
6634 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
6635 StringRef Format;
6636 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
6637 Format = S->getString();
6638
6639 // The "float-abi" key is now reserved for the target-independent
6640 // soft/hard ABI flag, so leave a valid value alone. Map any other value
6641 // (including unrecognized ones, which were never valid) to the default.
6642 if (!FloatABI::parseABIType(Format)) {
6643 LongDoubleFormat NewFormat =
6644 StringSwitch<LongDoubleFormat>(Format)
6645 .Case("ieeequad", LongDoubleFormat::IEEEquad)
6646 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
6647 .Default(LongDoubleFormat::PPCDoubleDouble);
6648 Metadata *Ops[3] = {
6649 Op->getOperand(0),
6650 MDString::get(M.getContext(), "long-double-type"),
6651 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
6652 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6653 Changed = true;
6654 }
6655 }
6656 }
6657
6658 // "Objective-C Class Properties" is recently added for Objective-C. We
6659 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
6660 // flag of value 0, so we can correclty downgrade this flag when trying to
6661 // link an ObjC bitcode without this module flag with an ObjC bitcode with
6662 // this module flag.
6663 if (HasObjCFlag && !HasClassProperties) {
6664 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
6665 (uint32_t)0);
6666 Changed = true;
6667 }
6668
6669 if (HasSwiftVersionFlag) {
6670 M.addModuleFlag(Module::Error, "Swift ABI Version",
6671 SwiftABIVersion);
6672 M.addModuleFlag(Module::Error, "Swift Major Version",
6673 ConstantInt::get(Int8Ty, SwiftMajorVersion));
6674 M.addModuleFlag(Module::Error, "Swift Minor Version",
6675 ConstantInt::get(Int8Ty, SwiftMinorVersion));
6676 Changed = true;
6677 }
6678
6679 return Changed;
6680}
6681
6682bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
6683 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
6684 // If this metadata has operands, we expect all of them to be either from
6685 // before or from after the format change handled here, so we can bail out
6686 // fast if the first (if any) operands is of the new format.
6687 auto MatchesVersion = [](const MDNode *Op) {
6688 return Op->getNumOperands() >= 3 &&
6689 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
6690 cast<ConstantAsMetadata>(Op->getOperand(2))
6691 ->getType()
6692 ->isIntegerTy(64);
6693 };
6694
6695 if (!CFIConsts || !CFIConsts->getNumOperands() ||
6696 MatchesVersion(CFIConsts->getOperand(0)))
6697 return false;
6698
6699 bool Changed = false;
6700 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
6701 MDNode *Op = CFIConsts->getOperand(I);
6702 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
6703 assert(Op->getNumOperands() >= 2 &&
6704 "Expected at least 2 operands - name and linkage type");
6705 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
6706 StringRef Name = NameMD->getString();
6707 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
6708 GlobalValue::dropLLVMManglingEscape(Name));
6709
6710 SmallVector<Metadata *, 4> Elts;
6711 Elts.push_back(Op->getOperand(0));
6712 Elts.push_back(Op->getOperand(1));
6713 Elts.push_back(ConstantAsMetadata::get(
6714 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
6715
6716 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
6717 Elts.push_back(Op->getOperand(J));
6718
6719 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
6720 Changed = true;
6721 }
6722
6723 return Changed;
6724}
6725
6726void llvm::UpgradeSectionAttributes(Module &M) {
6727 auto TrimSpaces = [](StringRef Section) -> std::string {
6728 SmallVector<StringRef, 5> Components;
6729 Section.split(Components, ',');
6730
6731 SmallString<32> Buffer;
6732 raw_svector_ostream OS(Buffer);
6733
6734 for (auto Component : Components)
6735 OS << ',' << Component.trim();
6736
6737 return std::string(OS.str().substr(1));
6738 };
6739
6740 for (auto &GV : M.globals()) {
6741 if (!GV.hasSection())
6742 continue;
6743
6744 StringRef Section = GV.getSection();
6745
6746 if (!Section.starts_with("__DATA, __objc_catlist"))
6747 continue;
6748
6749 // __DATA, __objc_catlist, regular, no_dead_strip
6750 // __DATA,__objc_catlist,regular,no_dead_strip
6751 GV.setSection(TrimSpaces(Section));
6752 }
6753}
6754
6755namespace {
6756// Prior to LLVM 10.0, the strictfp attribute could be used on individual
6757// callsites within a function that did not also have the strictfp attribute.
6758// Since 10.0, if strict FP semantics are needed within a function, the
6759// function must have the strictfp attribute and all calls within the function
6760// must also have the strictfp attribute. This latter restriction is
6761// necessary to prevent unwanted libcall simplification when a function is
6762// being cloned (such as for inlining).
6763//
6764// The "dangling" strictfp attribute usage was only used to prevent constant
6765// folding and other libcall simplification. The nobuiltin attribute on the
6766// callsite has the same effect.
6767struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
6768 StrictFPUpgradeVisitor() = default;
6769
6770 void visitCallBase(CallBase &Call) {
6771 if (!Call.isStrictFP())
6772 return;
6773 if (isa<ConstrainedFPIntrinsic>(&Call))
6774 return;
6775 // If we get here, the caller doesn't have the strictfp attribute
6776 // but this callsite does. Replace the strictfp attribute with nobuiltin.
6777 Call.removeFnAttr(Attribute::StrictFP);
6778 Call.addFnAttr(Attribute::NoBuiltin);
6779 }
6780};
6781
6782/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
6783struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
6784 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
6785 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
6786
6787 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
6788 if (!RMW.isFloatingPointOperation())
6789 return;
6790
6791 MDNode *Empty = MDNode::get(RMW.getContext(), {});
6792 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
6793 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
6794 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
6795 }
6796};
6797} // namespace
6798
6799void llvm::UpgradeFunctionAttributes(Function &F) {
6800 // If a function definition doesn't have the strictfp attribute,
6801 // convert any callsite strictfp attributes to nobuiltin.
6802 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
6803 StrictFPUpgradeVisitor SFPV;
6804 SFPV.visit(F);
6805 }
6806
6807 // Remove all incompatibile attributes from function.
6808 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
6809 F.getReturnType(), F.getAttributes().getRetAttrs()));
6810 for (auto &Arg : F.args())
6811 Arg.removeAttrs(
6812 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
6813
6814 bool AddingAttrs = false, RemovingAttrs = false;
6815 AttrBuilder AttrsToAdd(F.getContext());
6816 AttributeMask AttrsToRemove;
6817
6818 // Older versions of LLVM treated an "implicit-section-name" attribute
6819 // similarly to directly setting the section on a Function.
6820 if (Attribute A = F.getFnAttribute("implicit-section-name");
6821 A.isValid() && A.isStringAttribute()) {
6822 F.setSection(A.getValueAsString());
6823 AttrsToRemove.addAttribute("implicit-section-name");
6824 RemovingAttrs = true;
6825 }
6826
6827 if (Attribute A = F.getFnAttribute("nooutline");
6828 A.isValid() && A.isStringAttribute()) {
6829 AttrsToRemove.addAttribute("nooutline");
6830 AttrsToAdd.addAttribute(Attribute::NoOutline);
6831 AddingAttrs = RemovingAttrs = true;
6832 }
6833
6834 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
6835 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
6836 AttrsToRemove.addAttribute("uniform-work-group-size");
6837 RemovingAttrs = true;
6838 if (A.getValueAsString() == "true") {
6839 AttrsToAdd.addAttribute("uniform-work-group-size");
6840 AddingAttrs = true;
6841 }
6842 }
6843
6844 if (!F.empty()) {
6845 // For some reason this is called twice, and the first time is before any
6846 // instructions are loaded into the body.
6847
6848 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
6849 A.isValid()) {
6850
6851 if (A.getValueAsBool()) {
6852 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
6853 Visitor.visit(F);
6854 }
6855
6856 // We will leave behind dead attribute uses on external declarations, but
6857 // clang never added these to declarations anyway.
6858 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
6859 RemovingAttrs = true;
6860 }
6861 }
6862
6863 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
6864 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
6865
6866 bool HandleDenormalMode = false;
6867
6868 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
6869 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6870 if (ParsedMode.isValid()) {
6871 DenormalFPMath = ParsedMode;
6872 AttrsToRemove.addAttribute("denormal-fp-math");
6873 AddingAttrs = RemovingAttrs = true;
6874 HandleDenormalMode = true;
6875 }
6876 }
6877
6878 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
6879 Attr.isValid()) {
6880 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6881 if (ParsedMode.isValid()) {
6882 DenormalFPMathF32 = ParsedMode;
6883 AttrsToRemove.addAttribute("denormal-fp-math-f32");
6884 AddingAttrs = RemovingAttrs = true;
6885 HandleDenormalMode = true;
6886 }
6887 }
6888
6889 if (HandleDenormalMode)
6890 AttrsToAdd.addDenormalFPEnvAttr(
6891 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
6892
6893 if (RemovingAttrs)
6894 F.removeFnAttrs(AttrsToRemove);
6895
6896 if (AddingAttrs)
6897 F.addFnAttrs(AttrsToAdd);
6898}
6899
6900// Check if the function attribute is not present and set it.
6901static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
6902 StringRef Value) {
6903 if (!F.hasFnAttribute(FnAttrName))
6904 F.addFnAttr(FnAttrName, Value);
6905}
6906
6907// Check if the function attribute is not present and set it if needed.
6908// If the attribute is "false" then removes it.
6909// If the attribute is "true" resets it to a valueless attribute.
6910static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
6911 if (!F.hasFnAttribute(FnAttrName)) {
6912 if (Set)
6913 F.addFnAttr(FnAttrName);
6914 } else {
6915 auto A = F.getFnAttribute(FnAttrName);
6916 if ("false" == A.getValueAsString())
6917 F.removeFnAttr(FnAttrName);
6918 else if ("true" == A.getValueAsString()) {
6919 F.removeFnAttr(FnAttrName);
6920 F.addFnAttr(FnAttrName);
6921 }
6922 }
6923}
6924
6925void llvm::copyModuleAttrToFunctions(Module &M) {
6926 Triple T(M.getTargetTriple());
6927 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
6928 return;
6929
6930 uint64_t BTEValue = 0;
6931 uint64_t BPPLRValue = 0;
6932 uint64_t GCSValue = 0;
6933 uint64_t SRAValue = 0;
6934 uint64_t SRAALLValue = 0;
6935 uint64_t SRABKeyValue = 0;
6936
6937 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6938 if (ModFlags) {
6939 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6940 MDNode *Op = ModFlags->getOperand(I);
6941 if (Op->getNumOperands() != 3)
6942 continue;
6943
6944 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6945 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
6946 if (!ID || !CI)
6947 continue;
6948
6949 StringRef IDStr = ID->getString();
6950 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
6951 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
6952 : IDStr == "guarded-control-stack" ? &GCSValue
6953 : IDStr == "sign-return-address" ? &SRAValue
6954 : IDStr == "sign-return-address-all" ? &SRAALLValue
6955 : IDStr == "sign-return-address-with-bkey"
6956 ? &SRABKeyValue
6957 : nullptr;
6958 if (!ValPtr)
6959 continue;
6960
6961 *ValPtr = CI->getZExtValue();
6962 if (*ValPtr == 2)
6963 return;
6964 }
6965 }
6966
6967 bool BTE = BTEValue == 1;
6968 bool BPPLR = BPPLRValue == 1;
6969 bool GCS = GCSValue == 1;
6970 bool SRA = SRAValue == 1;
6971
6972 StringRef SignTypeValue = "non-leaf";
6973 if (SRA && SRAALLValue == 1)
6974 SignTypeValue = "all";
6975
6976 StringRef SignKeyValue = "a_key";
6977 if (SRA && SRABKeyValue == 1)
6978 SignKeyValue = "b_key";
6979
6980 for (Function &F : M.getFunctionList()) {
6981 if (F.isDeclaration())
6982 continue;
6983
6984 if (SRA) {
6985 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
6986 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
6987 } else {
6988 if (auto A = F.getFnAttribute("sign-return-address");
6989 A.isValid() && "none" == A.getValueAsString()) {
6990 F.removeFnAttr("sign-return-address");
6991 F.removeFnAttr("sign-return-address-key");
6992 }
6993 }
6994 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
6995 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
6996 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
6997 }
6998
6999 if (BTE)
7000 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7001 if (BPPLR)
7002 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7003 if (GCS)
7004 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7005 if (SRA) {
7006 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7007 if (SRAALLValue == 1)
7008 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7009 if (SRABKeyValue == 1)
7010 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7011 }
7012}
7013
7014// Old two-operand form: !{!"llvm.loop.distribute.enable", i1 X}. The new
7015// single-operand form uses "llvm.loop.distribute.enable" for X = true and
7016// "llvm.loop.distribute.disable" for X = false.
7017static bool isOldDistributeEnable(const MDTuple *T) {
7018 if (T->getNumOperands() != 2)
7019 return false;
7020 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7021 if (!Tag || Tag->getString() != "llvm.loop.distribute.enable")
7022 return false;
7023 return mdconst::hasa<ConstantInt>(T->getOperand(1));
7024}
7025
7026/// Old two-operand form: !{!"llvm.loop.vectorize.enable", i1 X}. The new
7027/// single-operand form uses "llvm.loop.vectorize.enable" for X = true and
7028/// "llvm.loop.vectorize.disable" for X = false.
7029static bool isOldVectorizeEnable(const MDTuple *T) {
7030 if (T->getNumOperands() != 2)
7031 return false;
7032 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7033 if (!Tag || Tag->getString() != "llvm.loop.vectorize.enable")
7034 return false;
7035 return mdconst::hasa<ConstantInt>(T->getOperand(1));
7036}
7037
7038/// Build the single-operand vectorize enable/disable node that replaces a
7039/// boolean operand: nonzero -> enable, zero -> disable.
7040static Metadata *makeVectorizeEnableNode(LLVMContext &C, const MDOperand &Op) {
7041 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7042 return MDTuple::get(
7043 C, {MDString::get(C, Enable ? "llvm.loop.vectorize.enable"
7044 : "llvm.loop.vectorize.disable")});
7045}
7046
7047static bool isOldLoopArgument(Metadata *MD) {
7048 auto *T = dyn_cast_or_null<MDTuple>(MD);
7049 if (!T)
7050 return false;
7051 if (T->getNumOperands() < 1)
7052 return false;
7053 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7054 if (!S)
7055 return false;
7056 if (S->getString().starts_with("llvm.vectorizer."))
7057 return true;
7058 return isOldDistributeEnable(T) || isOldVectorizeEnable(T);
7059}
7060
7061static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7062 StringRef OldPrefix = "llvm.vectorizer.";
7063 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7064
7065 if (OldTag == "llvm.vectorizer.unroll")
7066 return MDString::get(C, "llvm.loop.interleave.count");
7067
7068 return MDString::get(
7069 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7070 .str());
7071}
7072
7073static Metadata *upgradeLoopArgument(Metadata *MD) {
7074 auto *T = dyn_cast_or_null<MDTuple>(MD);
7075 if (!T)
7076 return MD;
7077 if (T->getNumOperands() < 1)
7078 return MD;
7079 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7080 if (!OldTag)
7081 return MD;
7082
7083 LLVMContext &C = T->getContext();
7084
7085 // Rewrite the old two-operand distribute form to the single-operand pair.
7086 if (isOldDistributeEnable(T)) {
7087 bool Enable = !mdconst::extract<ConstantInt>(T->getOperand(1))->isZero();
7088 return MDTuple::get(
7089 C, {MDString::get(C, Enable ? "llvm.loop.distribute.enable"
7090 : "llvm.loop.distribute.disable")});
7091 }
7092
7093 // Rewrite the modern two-operand vectorize.enable form to the single-operand
7094 // enable/disable pair.
7095 if (isOldVectorizeEnable(T))
7096 return makeVectorizeEnableNode(C, T->getOperand(1));
7097
7098 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7099 return MD;
7100
7101 // This has an old tag. Upgrade it.
7102 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7103
7104 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7105 // vectorize.enable/disable pair, not a two-operand enable node.
7106 if (NewTag->getString() == "llvm.loop.vectorize.enable" &&
7107 T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7108 return makeVectorizeEnableNode(C, T->getOperand(1));
7109
7110 SmallVector<Metadata *, 8> Ops;
7111 Ops.reserve(T->getNumOperands());
7112 Ops.push_back(NewTag);
7113 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7114 Ops.push_back(T->getOperand(I));
7115
7116 return MDTuple::get(C, Ops);
7117}
7118
7119MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7120 auto *T = dyn_cast<MDTuple>(&N);
7121 if (!T)
7122 return &N;
7123
7124 if (none_of(T->operands(), isOldLoopArgument))
7125 return &N;
7126
7127 // Fix the old two-operand distribute/vectorize.enable nodes in place: the
7128 // Verifier rejects any MDNode carrying those tags with more than one operand,
7129 // so a leftover reference (from the distinct loop-ID) would still trigger a
7130 // diagnostic. In-place mutation is safe on distinct MDNodes.
7131 if (T->isDistinct()) {
7132 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7133 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7134 if (OpT && (isOldDistributeEnable(OpT) || isOldVectorizeEnable(OpT)))
7135 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7136 }
7137 if (none_of(T->operands(), isOldLoopArgument))
7138 return &N;
7139 }
7140
7141 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7142 // attachment; the original distinct loop-ID is kept as the first operand.
7143 SmallVector<Metadata *, 8> Ops;
7144 Ops.reserve(T->getNumOperands());
7145 for (Metadata *MD : T->operands())
7146 Ops.push_back(upgradeLoopArgument(MD));
7147
7148 return MDTuple::get(T->getContext(), Ops);
7149}
7150
7151std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7152 Triple T(TT);
7153 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7154 // the address space of globals to 1. This does not apply to SPIRV Logical.
7155 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7156 !DL.contains("-G") && !DL.starts_with("G")) {
7157 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7158 }
7159
7160 if (T.isLoongArch64() || T.isRISCV64()) {
7161 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7162 auto I = DL.find("-n64-");
7163 if (I != StringRef::npos)
7164 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7165 return DL.str();
7166 }
7167
7168 // AMDGPU data layout upgrades.
7169 std::string Res = DL.str();
7170 if (T.isAMDGPU()) {
7171 // Define address spaces for constants.
7172 if (!DL.contains("-G") && !DL.starts_with("G"))
7173 Res.append(Res.empty() ? "G1" : "-G1");
7174
7175 // AMDGCN data layout upgrades.
7176 if (T.isAMDGCN()) {
7177
7178 // Add missing non-integral declarations.
7179 // This goes before adding new address spaces to prevent incoherent string
7180 // values.
7181 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7182 Res.append("-ni:7:8:9");
7183 // Update ni:7 to ni:7:8:9.
7184 if (DL.ends_with("ni:7"))
7185 Res.append(":8:9");
7186 if (DL.ends_with("ni:7:8"))
7187 Res.append(":9");
7188
7189 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7190 // resources) An empty data layout has already been upgraded to G1 by now.
7191 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7192 Res.append("-p7:160:256:256:32");
7193 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7194 Res.append("-p8:128:128:128:48");
7195 constexpr StringRef OldP8("-p8:128:128-");
7196 if (DL.contains(OldP8))
7197 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7198 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7199 Res.append("-p9:192:256:256:32");
7200 }
7201
7202 // Upgrade the ELF mangling mode.
7203 if (!DL.contains("m:e"))
7204 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7205
7206 return Res;
7207 }
7208
7209 if (T.isSystemZ() && !DL.empty()) {
7210 // Make sure the stack alignment is present.
7211 if (!DL.contains("-S64"))
7212 return "E-S64" + DL.drop_front(1).str();
7213 return DL.str();
7214 }
7215
7216 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7217 // If the datalayout matches the expected format, add pointer size address
7218 // spaces to the datalayout.
7219 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7220 if (!DL.contains(AddrSpaces)) {
7221 SmallVector<StringRef, 4> Groups;
7222 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7223 if (R.match(Res, &Groups))
7224 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7225 }
7226 };
7227
7228 // AArch64 data layout upgrades.
7229 if (T.isAArch64()) {
7230 // Add "-Fn32"
7231 if (!DL.empty() && !DL.contains("-Fn32"))
7232 Res.append("-Fn32");
7233 AddPtr32Ptr64AddrSpaces();
7234 return Res;
7235 }
7236
7237 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7238 T.isWasm()) {
7239 // Mips64 with o32 ABI did not add "-i128:128".
7240 // Add "-i128:128"
7241 std::string I64 = "-i64:64";
7242 std::string I128 = "-i128:128";
7243 if (!StringRef(Res).contains(I128)) {
7244 size_t Pos = Res.find(I64);
7245 if (Pos != size_t(-1))
7246 Res.insert(Pos + I64.size(), I128);
7247 }
7248 }
7249
7250 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7251 size_t Pos = Res.find("-S128");
7252 if (Pos == StringRef::npos)
7253 Pos = Res.size();
7254 Res.insert(Pos, "-f64:32:64");
7255 }
7256
7257 if (!T.isX86())
7258 return Res;
7259
7260 AddPtr32Ptr64AddrSpaces();
7261
7262 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7263 // for i128 operations prior to this being reflected in the data layout, and
7264 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7265 // boundaries, so although this is a breaking change, the upgrade is expected
7266 // to fix more IR than it breaks.
7267 // Intel MCU is an exception and uses 4-byte-alignment.
7268 if (!T.isOSIAMCU()) {
7269 std::string I128 = "-i128:128";
7270 if (StringRef Ref = Res; !Ref.contains(I128)) {
7271 SmallVector<StringRef, 4> Groups;
7272 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7273 if (R.match(Res, &Groups))
7274 Res = (Groups[1] + I128 + Groups[3]).str();
7275 }
7276 }
7277
7278 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7279 // Raising the alignment is safe because Clang did not produce f80 values in
7280 // the MSVC environment before this upgrade was added.
7281 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7282 StringRef Ref = Res;
7283 auto I = Ref.find("-f80:32-");
7284 if (I != StringRef::npos)
7285 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7286 }
7287
7288 return Res;
7289}
7290
7291void llvm::UpgradeAttributes(AttrBuilder &B) {
7292 StringRef FramePointer;
7293 Attribute A = B.getAttribute("no-frame-pointer-elim");
7294 if (A.isValid()) {
7295 // The value can be "true" or "false".
7296 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7297 B.removeAttribute("no-frame-pointer-elim");
7298 }
7299 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7300 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7301 if (FramePointer != "all")
7302 FramePointer = "non-leaf";
7303 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7304 }
7305 if (!FramePointer.empty())
7306 B.addAttribute("frame-pointer", FramePointer);
7307
7308 A = B.getAttribute("null-pointer-is-valid");
7309 if (A.isValid()) {
7310 // The value can be "true" or "false".
7311 bool NullPointerIsValid = A.getValueAsString() == "true";
7312 B.removeAttribute("null-pointer-is-valid");
7313 if (NullPointerIsValid)
7314 B.addAttribute(Attribute::NullPointerIsValid);
7315 }
7316
7317 A = B.getAttribute("uniform-work-group-size");
7318 if (A.isValid()) {
7319 StringRef Val = A.getValueAsString();
7320 if (!Val.empty()) {
7321 bool IsTrue = Val == "true";
7322 B.removeAttribute("uniform-work-group-size");
7323 if (IsTrue)
7324 B.addAttribute("uniform-work-group-size");
7325 }
7326 }
7327}
7328
7329void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7330 // clang.arc.attachedcall bundles are now required to have an operand.
7331 // If they don't, it's okay to drop them entirely: when there is an operand,
7332 // the "attachedcall" is meaningful and required, but without an operand,
7333 // it's just a marker NOP. Dropping it merely prevents an optimization.
7334 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7335 return OBD.getTag() == "clang.arc.attachedcall" &&
7336 OBD.inputs().empty();
7337 });
7338}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1312
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2214
makeVectorizeEnableNode
static Metadata * makeVectorizeEnableNode(LLVMContext &C, const MDOperand &Op)
Build the single-operand vectorize enable/disable node that replaces a boolean operand: nonzero -> en...
Definition AutoUpgrade.cpp:7040
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7073
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6150
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1342
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2054
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1195
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6256
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2315
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2541
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1292
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2289
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:125
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:6901
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1220
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6154
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5029
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7061
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6117
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:112
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2169
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2426
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2140
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4688
isOldDistributeEnable
static bool isOldDistributeEnable(const MDTuple *T)
Definition AutoUpgrade.cpp:7017
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2527
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:550
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2466
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6063
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1286
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:179
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2088
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1148
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7047
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4758
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:96
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5099
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4848
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2405
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5041
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:159
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:788
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5129
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2120
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2152
isOldVectorizeEnable
static bool isOldVectorizeEnable(const MDTuple *T)
Old two-operand form: !
Definition AutoUpgrade.cpp:7029
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2354
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:67
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:80
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:169
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2488
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2274
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2798
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2520
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2550
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5051
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:6910
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2436
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:71
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2387
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5112
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:143
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6411
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2996
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3391
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:81
LHS
Value * LHS
Definition X86PartialReduction.cpp:80
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:760
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:920
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:772
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:776
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:797
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:836
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:790
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:780
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:786
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:840
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:828
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:788
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:808
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:804
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:832
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:967
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:507
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:355
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1581
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1701
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3103
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3386
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3472
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2291
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:168
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:211
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:246
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:166
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:448
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:878
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:216
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:863
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2893
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:548
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1792
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:712
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1511
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:310
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:190
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:633
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:615
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:111
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:117
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:138
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:120
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:152
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:149
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1576
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1568
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1564
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1583
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1574
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2081
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:386
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:532
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:977
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:702
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1236
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:331
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:823
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5262
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint64_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2356
Changed
Changed
Definition ObjCARCOpts.cpp:2366
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1418
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:758
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1399
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1392
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:578
llvm::Length
@ Length
Definition DWP.cpp:578
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5184
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:6726
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:2789
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2002
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7119
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7291
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:5976
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6215
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6510
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:6682
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:6925
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7329
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6043
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7151
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2021
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:709
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6021
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6073
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:6799
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:5995
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6280
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7712
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0