LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstVisitor.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsAArch64.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsARM.h"
39#include "llvm/IR/IntrinsicsNVPTX.h"
40#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/IR/IntrinsicsWebAssembly.h"
42#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/MDBuilder.h"
45#include "llvm/IR/Metadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/NVVMIntrinsicUtils.h"
48#include "llvm/IR/Value.h"
49#include "llvm/IR/Verifier.h"
50#include "llvm/Support/AMDGPUAddrSpace.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/NVPTXAddrSpace.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/Regex.h"
57#include "llvm/Support/TimeProfiler.h"
58#include "llvm/TargetParser/Triple.h"
59#include <cstdint>
60#include <cstring>
61#include <numeric>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "auto-upgrade"
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(0);
85 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(F);
90 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(F);
106 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(F);
119 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
134 return false;
135
136 rename(F);
137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
152 return false;
153
154 rename(F);
155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(F);
165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(F);
175 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front("avx."))
186 return (Name.starts_with("blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with("movnt.") || // Added in 3.2
191 Name.starts_with("sqrt.p") || // Added in 7.0
192 Name.starts_with("storeu.") || // Added in 3.9
193 Name.starts_with("vbroadcast.s") || // Added in 3.5
194 Name.starts_with("vbroadcastf128") || // Added in 4.0
195 Name.starts_with("vextractf128.") || // Added in 3.7
196 Name.starts_with("vinsertf128.") || // Added in 3.7
197 Name.starts_with("vperm2f128.") || // Added in 6.0
198 Name.starts_with("vpermil.")); // Added in 3.1
199
200 if (Name.consume_front("avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with("pabs.") || // Added in 6.0
203 Name.starts_with("padds.") || // Added in 8.0
204 Name.starts_with("paddus.") || // Added in 8.0
205 Name.starts_with("pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with("pbroadcast") || // Added in 3.8
208 Name.starts_with("pcmpeq.") || // Added in 3.1
209 Name.starts_with("pcmpgt.") || // Added in 3.1
210 Name.starts_with("pmax") || // Added in 3.9
211 Name.starts_with("pmin") || // Added in 3.9
212 Name.starts_with("pmovsx") || // Added in 3.9
213 Name.starts_with("pmovzx") || // Added in 3.9
214 Name.starts_with("pmulh.w") || // Added in 24.0
215 Name.starts_with("pmulhu.w") || // Added in 24.0
216 Name == "pmul.dq" || // Added in 7.0
217 Name == "pmulu.dq" || // Added in 7.0
218 Name.starts_with("psll.dq") || // Added in 3.7
219 Name.starts_with("psrl.dq") || // Added in 3.7
220 Name.starts_with("psubs.") || // Added in 8.0
221 Name.starts_with("psubus.") || // Added in 8.0
222 Name.starts_with("vbroadcast") || // Added in 3.8
223 Name == "vbroadcasti128" || // Added in 3.7
224 Name == "vextracti128" || // Added in 3.7
225 Name == "vinserti128" || // Added in 3.7
226 Name == "vperm2i128"); // Added in 6.0
227
228 if (Name.consume_front("avx512.")) {
229 if (Name.consume_front("mask."))
230 // 'avx512.mask.*'
231 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
232 Name.starts_with("and.") || // Added in 3.9
233 Name.starts_with("andn.") || // Added in 3.9
234 Name.starts_with("broadcast.s") || // Added in 3.9
235 Name.starts_with("broadcastf32x4.") || // Added in 6.0
236 Name.starts_with("broadcastf32x8.") || // Added in 6.0
237 Name.starts_with("broadcastf64x2.") || // Added in 6.0
238 Name.starts_with("broadcastf64x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x4.") || // Added in 6.0
240 Name.starts_with("broadcasti32x8.") || // Added in 6.0
241 Name.starts_with("broadcasti64x2.") || // Added in 6.0
242 Name.starts_with("broadcasti64x4.") || // Added in 6.0
243 Name.starts_with("cmp.b") || // Added in 5.0
244 Name.starts_with("cmp.d") || // Added in 5.0
245 Name.starts_with("cmp.q") || // Added in 5.0
246 Name.starts_with("cmp.w") || // Added in 5.0
247 Name.starts_with("compress.b") || // Added in 9.0
248 Name.starts_with("compress.d") || // Added in 9.0
249 Name.starts_with("compress.p") || // Added in 9.0
250 Name.starts_with("compress.q") || // Added in 9.0
251 Name.starts_with("compress.store.") || // Added in 7.0
252 Name.starts_with("compress.w") || // Added in 9.0
253 Name.starts_with("conflict.") || // Added in 9.0
254 Name.starts_with("cvtdq2pd.") || // Added in 4.0
255 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
256 Name == "cvtpd2dq.256" || // Added in 7.0
257 Name == "cvtpd2ps.256" || // Added in 7.0
258 Name == "cvtps2pd.128" || // Added in 7.0
259 Name == "cvtps2pd.256" || // Added in 7.0
260 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
261 Name == "cvtqq2ps.256" || // Added in 9.0
262 Name == "cvtqq2ps.512" || // Added in 9.0
263 Name == "cvttpd2dq.256" || // Added in 7.0
264 Name == "cvttps2dq.128" || // Added in 7.0
265 Name == "cvttps2dq.256" || // Added in 7.0
266 Name.starts_with("cvtudq2pd.") || // Added in 4.0
267 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
268 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
269 Name == "cvtuqq2ps.256" || // Added in 9.0
270 Name == "cvtuqq2ps.512" || // Added in 9.0
271 Name.starts_with("dbpsadbw.") || // Added in 7.0
272 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
273 Name.starts_with("expand.b") || // Added in 9.0
274 Name.starts_with("expand.d") || // Added in 9.0
275 Name.starts_with("expand.load.") || // Added in 7.0
276 Name.starts_with("expand.p") || // Added in 9.0
277 Name.starts_with("expand.q") || // Added in 9.0
278 Name.starts_with("expand.w") || // Added in 9.0
279 Name.starts_with("fpclass.p") || // Added in 7.0
280 Name.starts_with("insert") || // Added in 4.0
281 Name.starts_with("load.") || // Added in 3.9
282 Name.starts_with("loadu.") || // Added in 3.9
283 Name.starts_with("lzcnt.") || // Added in 5.0
284 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with("movddup") || // Added in 3.9
287 Name.starts_with("move.s") || // Added in 4.0
288 Name.starts_with("movshdup") || // Added in 3.9
289 Name.starts_with("movsldup") || // Added in 3.9
290 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
291 Name.starts_with("or.") || // Added in 3.9
292 Name.starts_with("pabs.") || // Added in 6.0
293 Name.starts_with("packssdw.") || // Added in 5.0
294 Name.starts_with("packsswb.") || // Added in 5.0
295 Name.starts_with("packusdw.") || // Added in 5.0
296 Name.starts_with("packuswb.") || // Added in 5.0
297 Name.starts_with("padd.") || // Added in 4.0
298 Name.starts_with("padds.") || // Added in 8.0
299 Name.starts_with("paddus.") || // Added in 8.0
300 Name.starts_with("palignr.") || // Added in 3.9
301 Name.starts_with("pand.") || // Added in 3.9
302 Name.starts_with("pandn.") || // Added in 3.9
303 Name.starts_with("pavg") || // Added in 6.0
304 Name.starts_with("pbroadcast") || // Added in 6.0
305 Name.starts_with("pcmpeq.") || // Added in 3.9
306 Name.starts_with("pcmpgt.") || // Added in 3.9
307 Name.starts_with("perm.df.") || // Added in 3.9
308 Name.starts_with("perm.di.") || // Added in 3.9
309 Name.starts_with("permvar.") || // Added in 7.0
310 Name.starts_with("pmaddubs.w.") || // Added in 7.0
311 Name.starts_with("pmaddw.d.") || // Added in 7.0
312 Name.starts_with("pmax") || // Added in 4.0
313 Name.starts_with("pmin") || // Added in 4.0
314 Name == "pmov.qd.256" || // Added in 9.0
315 Name == "pmov.qd.512" || // Added in 9.0
316 Name == "pmov.wb.256" || // Added in 9.0
317 Name == "pmov.wb.512" || // Added in 9.0
318 Name.starts_with("pmovsx") || // Added in 4.0
319 Name.starts_with("pmovzx") || // Added in 4.0
320 Name.starts_with("pmul.dq.") || // Added in 4.0
321 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
322 Name.starts_with("pmulh.w.") || // Added in 7.0
323 Name.starts_with("pmulhu.w.") || // Added in 7.0
324 Name.starts_with("pmull.") || // Added in 4.0
325 Name.starts_with("pmultishift.qb.") || // Added in 8.0
326 Name.starts_with("pmulu.dq.") || // Added in 4.0
327 Name.starts_with("por.") || // Added in 3.9
328 Name.starts_with("prol.") || // Added in 8.0
329 Name.starts_with("prolv.") || // Added in 8.0
330 Name.starts_with("pror.") || // Added in 8.0
331 Name.starts_with("prorv.") || // Added in 8.0
332 Name.starts_with("pshuf.b.") || // Added in 4.0
333 Name.starts_with("pshuf.d.") || // Added in 3.9
334 Name.starts_with("pshufh.w.") || // Added in 3.9
335 Name.starts_with("pshufl.w.") || // Added in 3.9
336 Name.starts_with("psll.d") || // Added in 4.0
337 Name.starts_with("psll.q") || // Added in 4.0
338 Name.starts_with("psll.w") || // Added in 4.0
339 Name.starts_with("pslli") || // Added in 4.0
340 Name.starts_with("psllv") || // Added in 4.0
341 Name.starts_with("psra.d") || // Added in 4.0
342 Name.starts_with("psra.q") || // Added in 4.0
343 Name.starts_with("psra.w") || // Added in 4.0
344 Name.starts_with("psrai") || // Added in 4.0
345 Name.starts_with("psrav") || // Added in 4.0
346 Name.starts_with("psrl.d") || // Added in 4.0
347 Name.starts_with("psrl.q") || // Added in 4.0
348 Name.starts_with("psrl.w") || // Added in 4.0
349 Name.starts_with("psrli") || // Added in 4.0
350 Name.starts_with("psrlv") || // Added in 4.0
351 Name.starts_with("psub.") || // Added in 4.0
352 Name.starts_with("psubs.") || // Added in 8.0
353 Name.starts_with("psubus.") || // Added in 8.0
354 Name.starts_with("pternlog.") || // Added in 7.0
355 Name.starts_with("punpckh") || // Added in 3.9
356 Name.starts_with("punpckl") || // Added in 3.9
357 Name.starts_with("pxor.") || // Added in 3.9
358 Name.starts_with("shuf.f") || // Added in 6.0
359 Name.starts_with("shuf.i") || // Added in 6.0
360 Name.starts_with("shuf.p") || // Added in 4.0
361 Name.starts_with("sqrt.p") || // Added in 7.0
362 Name.starts_with("store.b.") || // Added in 3.9
363 Name.starts_with("store.d.") || // Added in 3.9
364 Name.starts_with("store.p") || // Added in 3.9
365 Name.starts_with("store.q.") || // Added in 3.9
366 Name.starts_with("store.w.") || // Added in 3.9
367 Name == "store.ss" || // Added in 7.0
368 Name.starts_with("storeu.") || // Added in 3.9
369 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
370 Name.starts_with("ucmp.") || // Added in 5.0
371 Name.starts_with("unpckh.") || // Added in 3.9
372 Name.starts_with("unpckl.") || // Added in 3.9
373 Name.starts_with("valign.") || // Added in 4.0
374 Name == "vcvtph2ps.128" || // Added in 11.0
375 Name == "vcvtph2ps.256" || // Added in 11.0
376 Name.starts_with("vextract") || // Added in 4.0
377 Name.starts_with("vfmadd.") || // Added in 7.0
378 Name.starts_with("vfmaddsub.") || // Added in 7.0
379 Name.starts_with("vfnmadd.") || // Added in 7.0
380 Name.starts_with("vfnmsub.") || // Added in 7.0
381 Name.starts_with("vpdpbusd.") || // Added in 7.0
382 Name.starts_with("vpdpbusds.") || // Added in 7.0
383 Name.starts_with("vpdpwssd.") || // Added in 7.0
384 Name.starts_with("vpdpwssds.") || // Added in 7.0
385 Name.starts_with("vpermi2var.") || // Added in 7.0
386 Name.starts_with("vpermil.p") || // Added in 3.9
387 Name.starts_with("vpermilvar.") || // Added in 4.0
388 Name.starts_with("vpermt2var.") || // Added in 7.0
389 Name.starts_with("vpmadd52") || // Added in 7.0
390 Name.starts_with("vpshld.") || // Added in 7.0
391 Name.starts_with("vpshldv.") || // Added in 8.0
392 Name.starts_with("vpshrd.") || // Added in 7.0
393 Name.starts_with("vpshrdv.") || // Added in 8.0
394 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
395 Name.starts_with("xor.")); // Added in 3.9
396
397 if (Name.consume_front("mask3."))
398 // 'avx512.mask3.*'
399 return (Name.starts_with("vfmadd.") || // Added in 7.0
400 Name.starts_with("vfmaddsub.") || // Added in 7.0
401 Name.starts_with("vfmsub.") || // Added in 7.0
402 Name.starts_with("vfmsubadd.") || // Added in 7.0
403 Name.starts_with("vfnmsub.")); // Added in 7.0
404
405 if (Name.consume_front("maskz."))
406 // 'avx512.maskz.*'
407 return (Name.starts_with("pternlog.") || // Added in 7.0
408 Name.starts_with("vfmadd.") || // Added in 7.0
409 Name.starts_with("vfmaddsub.") || // Added in 7.0
410 Name.starts_with("vpdpbusd.") || // Added in 7.0
411 Name.starts_with("vpdpbusds.") || // Added in 7.0
412 Name.starts_with("vpdpwssd.") || // Added in 7.0
413 Name.starts_with("vpdpwssds.") || // Added in 7.0
414 Name.starts_with("vpermt2var.") || // Added in 7.0
415 Name.starts_with("vpmadd52") || // Added in 7.0
416 Name.starts_with("vpshldv.") || // Added in 8.0
417 Name.starts_with("vpshrdv.")); // Added in 8.0
418
419 // 'avx512.*'
420 return (Name == "movntdqa" || // Added in 5.0
421 Name == "pmul.dq.512" || // Added in 7.0
422 Name == "pmulu.dq.512" || // Added in 7.0
423 Name.starts_with("broadcastm") || // Added in 6.0
424 Name.starts_with("cmp.p") || // Added in 12.0
425 Name.starts_with("cvtb2mask.") || // Added in 7.0
426 Name.starts_with("cvtd2mask.") || // Added in 7.0
427 Name.starts_with("cvtmask2") || // Added in 5.0
428 Name.starts_with("cvtq2mask.") || // Added in 7.0
429 Name == "cvtusi2sd" || // Added in 7.0
430 Name.starts_with("cvtw2mask.") || // Added in 7.0
431 Name == "kand.w" || // Added in 7.0
432 Name == "kandn.w" || // Added in 7.0
433 Name == "knot.w" || // Added in 7.0
434 Name == "kor.w" || // Added in 7.0
435 Name == "kortestc.w" || // Added in 7.0
436 Name == "kortestz.w" || // Added in 7.0
437 Name.starts_with("kunpck") || // added in 6.0
438 Name == "kxnor.w" || // Added in 7.0
439 Name == "kxor.w" || // Added in 7.0
440 Name.starts_with("padds.") || // Added in 8.0
441 Name.starts_with("pbroadcast") || // Added in 3.9
442 Name.starts_with("pmulh.w") || // Added in 24.0
443 Name.starts_with("pmulhu.w") || // Added in 24.0
444 Name.starts_with("prol") || // Added in 8.0
445 Name.starts_with("pror") || // Added in 8.0
446 Name.starts_with("psll.dq") || // Added in 3.9
447 Name.starts_with("psrl.dq") || // Added in 3.9
448 Name.starts_with("psubs.") || // Added in 8.0
449 Name.starts_with("ptestm") || // Added in 6.0
450 Name.starts_with("ptestnm") || // Added in 6.0
451 Name.starts_with("storent.") || // Added in 3.9
452 Name.starts_with("vbroadcast.s") || // Added in 7.0
453 Name.starts_with("vpshld.") || // Added in 8.0
454 Name.starts_with("vpshrd.")); // Added in 8.0
455 }
456
457 if (Name.consume_front("fma."))
458 return (Name.starts_with("vfmadd.") || // Added in 7.0
459 Name.starts_with("vfmsub.") || // Added in 7.0
460 Name.starts_with("vfmsubadd.") || // Added in 7.0
461 Name.starts_with("vfnmadd.") || // Added in 7.0
462 Name.starts_with("vfnmsub.")); // Added in 7.0
463
464 if (Name.consume_front("fma4."))
465 return Name.starts_with("vfmadd.s"); // Added in 7.0
466
467 if (Name.consume_front("sse."))
468 return (Name == "add.ss" || // Added in 4.0
469 Name == "cvtsi2ss" || // Added in 7.0
470 Name == "cvtsi642ss" || // Added in 7.0
471 Name == "div.ss" || // Added in 4.0
472 Name == "mul.ss" || // Added in 4.0
473 Name.starts_with("sqrt.p") || // Added in 7.0
474 Name == "sqrt.ss" || // Added in 7.0
475 Name.starts_with("storeu.") || // Added in 3.9
476 Name == "sub.ss"); // Added in 4.0
477
478 if (Name.consume_front("sse2."))
479 return (Name == "add.sd" || // Added in 4.0
480 Name == "cvtdq2pd" || // Added in 3.9
481 Name == "cvtdq2ps" || // Added in 7.0
482 Name == "cvtps2pd" || // Added in 3.9
483 Name == "cvtsi2sd" || // Added in 7.0
484 Name == "cvtsi642sd" || // Added in 7.0
485 Name == "cvtss2sd" || // Added in 7.0
486 Name == "div.sd" || // Added in 4.0
487 Name == "mul.sd" || // Added in 4.0
488 Name.starts_with("padds.") || // Added in 8.0
489 Name.starts_with("paddus.") || // Added in 8.0
490 Name.starts_with("pcmpeq.") || // Added in 3.1
491 Name.starts_with("pcmpgt.") || // Added in 3.1
492 Name == "pmaxs.w" || // Added in 3.9
493 Name == "pmaxu.b" || // Added in 3.9
494 Name == "pmins.w" || // Added in 3.9
495 Name == "pminu.b" || // Added in 3.9
496 Name == "pmulh.w" || // Added in 24.0
497 Name == "pmulhu.w" || // Added in 24.0
498 Name == "pmulu.dq" || // Added in 7.0
499 Name.starts_with("pshuf") || // Added in 3.9
500 Name.starts_with("psll.dq") || // Added in 3.7
501 Name.starts_with("psrl.dq") || // Added in 3.7
502 Name.starts_with("psubs.") || // Added in 8.0
503 Name.starts_with("psubus.") || // Added in 8.0
504 Name.starts_with("sqrt.p") || // Added in 7.0
505 Name == "sqrt.sd" || // Added in 7.0
506 Name == "storel.dq" || // Added in 3.9
507 Name.starts_with("storeu.") || // Added in 3.9
508 Name == "sub.sd"); // Added in 4.0
509
510 if (Name.consume_front("sse41."))
511 return (Name.starts_with("blendp") || // Added in 3.7
512 Name == "movntdqa" || // Added in 5.0
513 Name == "pblendw" || // Added in 3.7
514 Name == "pmaxsb" || // Added in 3.9
515 Name == "pmaxsd" || // Added in 3.9
516 Name == "pmaxud" || // Added in 3.9
517 Name == "pmaxuw" || // Added in 3.9
518 Name == "pminsb" || // Added in 3.9
519 Name == "pminsd" || // Added in 3.9
520 Name == "pminud" || // Added in 3.9
521 Name == "pminuw" || // Added in 3.9
522 Name.starts_with("pmovsx") || // Added in 3.8
523 Name.starts_with("pmovzx") || // Added in 3.9
524 Name == "pmuldq"); // Added in 7.0
525
526 if (Name.consume_front("sse42."))
527 return Name == "crc32.64.8"; // Added in 3.4
528
529 if (Name.consume_front("sse4a."))
530 return Name.starts_with("movnt."); // Added in 3.9
531
532 if (Name.consume_front("ssse3."))
533 return (Name == "pabs.b.128" || // Added in 6.0
534 Name == "pabs.d.128" || // Added in 6.0
535 Name == "pabs.w.128"); // Added in 6.0
536
537 if (Name.consume_front("xop."))
538 return (Name == "vpcmov" || // Added in 3.8
539 Name == "vpcmov.256" || // Added in 5.0
540 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
541 Name.starts_with("vprot")); // Added in 8.0
542
543 if (Name.consume_front("bmi."))
544 return (Name.starts_with("pdep.") || // Added in 23.0
545 Name.starts_with("pext.")); // Added in 23.0
546
547 return (Name == "addcarry.u32" || // Added in 8.0
548 Name == "addcarry.u64" || // Added in 8.0
549 Name == "addcarryx.u32" || // Added in 8.0
550 Name == "addcarryx.u64" || // Added in 8.0
551 Name == "subborrow.u32" || // Added in 8.0
552 Name == "subborrow.u64" || // Added in 8.0
553 Name.starts_with("vcvtph2ps.")); // Added in 11.0
554}
555
556static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
557 Function *&NewFn) {
558 // Only handle intrinsics that start with "x86.".
559 if (!Name.consume_front("x86."))
560 return false;
561
562 if (shouldUpgradeX86Intrinsic(F, Name)) {
563 NewFn = nullptr;
564 return true;
565 }
566
567 if (Name == "rdtscp") { // Added in 8.0
568 // If this intrinsic has 0 operands, it's the new version.
569 if (F->getFunctionType()->getNumParams() == 0)
570 return false;
571
572 rename(F);
573 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
574 Intrinsic::x86_rdtscp);
575 return true;
576 }
577
578 Intrinsic::ID ID;
579
580 // SSE4.1 ptest functions may have an old signature.
581 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
582 ID = StringSwitch<Intrinsic::ID>(Name)
583 .Case("c", Intrinsic::x86_sse41_ptestc)
584 .Case("z", Intrinsic::x86_sse41_ptestz)
585 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
586 .Default(Intrinsic::not_intrinsic);
587 if (ID != Intrinsic::not_intrinsic)
588 return upgradePTESTIntrinsic(F, ID, NewFn);
589
590 return false;
591 }
592
593 // Several blend and other instructions with masks used the wrong number of
594 // bits.
595
596 // Added in 3.6
597 ID = StringSwitch<Intrinsic::ID>(Name)
598 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
599 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
600 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
601 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
602 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
603 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
604 .Default(Intrinsic::not_intrinsic);
605 if (ID != Intrinsic::not_intrinsic)
606 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
607
608 if (Name.consume_front("avx512.")) {
609 if (Name.consume_front("mask.cmp.")) {
610 // Added in 7.0
611 ID = StringSwitch<Intrinsic::ID>(Name)
612 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
613 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
614 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
615 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
616 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
617 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
618 .Default(Intrinsic::not_intrinsic);
619 if (ID != Intrinsic::not_intrinsic)
620 return upgradeX86MaskedFPCompare(F, ID, NewFn);
621 } else if (Name.starts_with("vpdpbusd.") ||
622 Name.starts_with("vpdpbusds.")) {
623 // Added in 21.1
624 ID = StringSwitch<Intrinsic::ID>(Name)
625 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
626 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
627 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
628 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
629 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
630 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
631 .Default(Intrinsic::not_intrinsic);
632 if (ID != Intrinsic::not_intrinsic)
633 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
634 } else if (Name.starts_with("vpdpwssd.") ||
635 Name.starts_with("vpdpwssds.")) {
636 // Added in 21.1
637 ID = StringSwitch<Intrinsic::ID>(Name)
638 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
639 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
640 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
641 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
642 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
643 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
644 .Default(Intrinsic::not_intrinsic);
645 if (ID != Intrinsic::not_intrinsic)
646 return upgradeX86MultiplyAddWords(F, ID, NewFn);
647 }
648 return false; // No other 'x86.avx512.*'.
649 }
650
651 if (Name.consume_front("avx2.")) {
652 if (Name.consume_front("vpdpb")) {
653 // Added in 21.1
654 ID = StringSwitch<Intrinsic::ID>(Name)
655 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
656 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
657 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
658 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
659 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
660 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
661 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
662 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
663 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
664 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
665 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
666 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
667 .Default(Intrinsic::not_intrinsic);
668 if (ID != Intrinsic::not_intrinsic)
669 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
670 } else if (Name.consume_front("vpdpw")) {
671 // Added in 21.1
672 ID = StringSwitch<Intrinsic::ID>(Name)
673 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
674 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
675 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
676 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
677 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
678 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
679 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
680 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
681 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
682 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
683 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
684 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
685 .Default(Intrinsic::not_intrinsic);
686 if (ID != Intrinsic::not_intrinsic)
687 return upgradeX86MultiplyAddWords(F, ID, NewFn);
688 }
689 return false; // No other 'x86.avx2.*'
690 }
691
692 if (Name.consume_front("avx10.")) {
693 if (Name.consume_front("vpdpb")) {
694 // Added in 21.1
695 ID = StringSwitch<Intrinsic::ID>(Name)
696 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
697 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
698 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
699 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
700 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
701 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
702 .Default(Intrinsic::not_intrinsic);
703 if (ID != Intrinsic::not_intrinsic)
704 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
705 } else if (Name.consume_front("vpdpw")) {
706 ID = StringSwitch<Intrinsic::ID>(Name)
707 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
708 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
709 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
710 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
711 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
712 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
713 .Default(Intrinsic::not_intrinsic);
714 if (ID != Intrinsic::not_intrinsic)
715 return upgradeX86MultiplyAddWords(F, ID, NewFn);
716 }
717 return false; // No other 'x86.avx10.*'
718 }
719
720 if (Name.consume_front("avx512bf16.")) {
721 // Added in 9.0
722 ID = StringSwitch<Intrinsic::ID>(Name)
723 .Case("cvtne2ps2bf16.128",
724 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
725 .Case("cvtne2ps2bf16.256",
726 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
727 .Case("cvtne2ps2bf16.512",
728 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
729 .Case("mask.cvtneps2bf16.128",
730 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
731 .Case("cvtneps2bf16.256",
732 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
733 .Case("cvtneps2bf16.512",
734 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
735 .Default(Intrinsic::not_intrinsic);
736 if (ID != Intrinsic::not_intrinsic)
737 return upgradeX86BF16Intrinsic(F, ID, NewFn);
738
739 // Added in 9.0
740 ID = StringSwitch<Intrinsic::ID>(Name)
741 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
742 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
743 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
744 .Default(Intrinsic::not_intrinsic);
745 if (ID != Intrinsic::not_intrinsic)
746 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
747 return false; // No other 'x86.avx512bf16.*'.
748 }
749
750 if (Name.consume_front("xop.")) {
751 Intrinsic::ID ID = Intrinsic::not_intrinsic;
752 if (Name.starts_with("vpermil2")) { // Added in 3.9
753 // Upgrade any XOP PERMIL2 index operand still using a float/double
754 // vector.
755 auto Idx = F->getFunctionType()->getParamType(2);
756 if (Idx->isFPOrFPVectorTy()) {
757 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
758 unsigned EltSize = Idx->getScalarSizeInBits();
759 if (EltSize == 64 && IdxSize == 128)
760 ID = Intrinsic::x86_xop_vpermil2pd;
761 else if (EltSize == 32 && IdxSize == 128)
762 ID = Intrinsic::x86_xop_vpermil2ps;
763 else if (EltSize == 64 && IdxSize == 256)
764 ID = Intrinsic::x86_xop_vpermil2pd_256;
765 else
766 ID = Intrinsic::x86_xop_vpermil2ps_256;
767 }
768 } else if (F->arg_size() == 2)
769 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
770 ID = StringSwitch<Intrinsic::ID>(Name)
771 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
772 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
773 .Default(Intrinsic::not_intrinsic);
774
775 if (ID != Intrinsic::not_intrinsic) {
776 rename(F);
777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
778 return true;
779 }
780 return false; // No other 'x86.xop.*'
781 }
782
783 if (Name == "seh.recoverfp") {
784 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
785 Intrinsic::eh_recoverfp);
786 return true;
787 }
788
789 return false;
790}
791
792// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
793// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
794static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
795 StringRef Name,
796 Function *&NewFn) {
797 if (Name.starts_with("rbit")) {
798 // '(arm|aarch64).rbit'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
801 return true;
802 }
803
804 if (Name == "thread.pointer") {
805 // '(arm|aarch64).thread.pointer'.
806 NewFn = Intrinsic::getOrInsertDeclaration(
807 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
808 return true;
809 }
810
811 bool Neon = Name.consume_front("neon.");
812 if (Neon) {
813 // '(arm|aarch64).neon.*'.
814 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
815 // v16i8 respectively.
816 if (Name.consume_front("bfdot.")) {
817 // (arm|aarch64).neon.bfdot.*'.
818 Intrinsic::ID ID =
819 StringSwitch<Intrinsic::ID>(Name)
820 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
821 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
822 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
823 .Default(Intrinsic::not_intrinsic);
824 if (ID != Intrinsic::not_intrinsic) {
825 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
826 assert((OperandWidth == 64 || OperandWidth == 128) &&
827 "Unexpected operand width");
828 LLVMContext &Ctx = F->getParent()->getContext();
829 std::array<Type *, 2> Tys{
830 {F->getReturnType(),
831 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
832 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
833 return true;
834 }
835 return false; // No other '(arm|aarch64).neon.bfdot.*'.
836 }
837
838 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
839 // anymore and accept v8bf16 instead of v16i8.
840 if (Name.consume_front("bfm")) {
841 // (arm|aarch64).neon.bfm*'.
842 if (Name.consume_back(".v4f32.v16i8")) {
843 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
844 Intrinsic::ID ID =
845 StringSwitch<Intrinsic::ID>(Name)
846 .Case("mla",
847 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
849 .Case("lalb",
850 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
851 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
852 .Case("lalt",
853 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
854 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
855 .Default(Intrinsic::not_intrinsic);
856 if (ID != Intrinsic::not_intrinsic) {
857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
858 return true;
859 }
860 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
861 }
862 return false; // No other '(arm|aarch64).neon.bfm*.
863 }
864 // Continue on to Aarch64 Neon or Arm Neon.
865 }
866 // Continue on to Arm or Aarch64.
867
868 if (IsArm) {
869 // 'arm.*'.
870 if (Neon) {
871 // 'arm.neon.*'.
872 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
873 .StartsWith("vclz.", Intrinsic::ctlz)
874 .StartsWith("vcnt.", Intrinsic::ctpop)
875 .StartsWith("vqadds.", Intrinsic::sadd_sat)
876 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
877 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
878 .StartsWith("vqsubu.", Intrinsic::usub_sat)
879 .StartsWith("vrinta.", Intrinsic::round)
880 .StartsWith("vrintn.", Intrinsic::roundeven)
881 .StartsWith("vrintm.", Intrinsic::floor)
882 .StartsWith("vrintp.", Intrinsic::ceil)
883 .StartsWith("vrintx.", Intrinsic::rint)
884 .StartsWith("vrintz.", Intrinsic::trunc)
885 .Default(Intrinsic::not_intrinsic);
886 if (ID != Intrinsic::not_intrinsic) {
887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
888 F->arg_begin()->getType());
889 return true;
890 }
891
892 if (Name.consume_front("vst")) {
893 // 'arm.neon.vst*'.
894 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
895 SmallVector<StringRef, 2> Groups;
896 if (vstRegex.match(Name, &Groups)) {
897 static const Intrinsic::ID StoreInts[] = {
898 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
899 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
900
901 static const Intrinsic::ID StoreLaneInts[] = {
902 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
903 Intrinsic::arm_neon_vst4lane};
904
905 auto fArgs = F->getFunctionType()->params();
906 Type *Tys[] = {fArgs[0], fArgs[1]};
907 if (Groups[1].size() == 1)
908 NewFn = Intrinsic::getOrInsertDeclaration(
909 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
910 else
911 NewFn = Intrinsic::getOrInsertDeclaration(
912 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
913 return true;
914 }
915 return false; // No other 'arm.neon.vst*'.
916 }
917
918 return false; // No other 'arm.neon.*'.
919 }
920
921 if (Name.consume_front("mve.")) {
922 // 'arm.mve.*'.
923 if (Name == "vctp64") {
924 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
925 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
926 // the function and deal with it below in UpgradeIntrinsicCall.
927 rename(F);
928 return true;
929 }
930 return false; // Not 'arm.mve.vctp64'.
931 }
932
933 if (Name.starts_with("vrintn.v")) {
934 NewFn = Intrinsic::getOrInsertDeclaration(
935 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
936 return true;
937 }
938
939 // These too are changed to accept a v2i1 instead of the old v4i1.
940 if (Name.consume_back(".v4i1")) {
941 // 'arm.mve.*.v4i1'.
942 if (Name.consume_back(".predicated.v2i64.v4i32"))
943 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
944 return Name == "mull.int" || Name == "vqdmull";
945
946 if (Name.consume_back(".v2i64")) {
947 // 'arm.mve.*.v2i64.v4i1'
948 bool IsGather = Name.consume_front("vldr.gather.");
949 if (IsGather || Name.consume_front("vstr.scatter.")) {
950 if (Name.consume_front("base.")) {
951 // Optional 'wb.' prefix.
952 Name.consume_front("wb.");
953 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
954 // predicated.v2i64.v2i64.v4i1'.
955 return Name == "predicated.v2i64";
956 }
957
958 if (Name.consume_front("offset.predicated."))
959 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
960 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
961
962 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
963 return false;
964 }
965
966 return false; // No other 'arm.mve.*.v2i64.v4i1'.
967 }
968 return false; // No other 'arm.mve.*.v4i1'.
969 }
970 return false; // No other 'arm.mve.*'.
971 }
972
973 if (Name.consume_front("cde.vcx")) {
974 // 'arm.cde.vcx*'.
975 if (Name.consume_back(".predicated.v2i64.v4i1"))
976 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
977 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
978 Name == "3q" || Name == "3qa";
979
980 return false; // No other 'arm.cde.vcx*'.
981 }
982 } else {
983 // 'aarch64.*'.
984 if (Neon) {
985 // 'aarch64.neon.*'.
986 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
987 .StartsWith("frintn", Intrinsic::roundeven)
988 .StartsWith("rbit", Intrinsic::bitreverse)
989 .Default(Intrinsic::not_intrinsic);
990 if (ID != Intrinsic::not_intrinsic) {
991 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
992 F->arg_begin()->getType());
993 return true;
994 }
995
996 Intrinsic::ID MinMaxID =
997 StringSwitch<Intrinsic::ID>(Name.split('.').first)
998 .Case("smax", Intrinsic::smax)
999 .Case("smin", Intrinsic::smin)
1000 .Case("umax", Intrinsic::umax)
1001 .Case("umin", Intrinsic::umin)
1002 .Default(Intrinsic::not_intrinsic);
1003 if (MinMaxID != Intrinsic::not_intrinsic) {
1004 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1005 return false; // Invalid IR.
1006 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1007 F->getReturnType());
1008 return true;
1009 }
1010
1011 if (Name.starts_with("addp")) {
1012 // 'aarch64.neon.addp*'.
1013 if (F->arg_size() != 2)
1014 return false; // Invalid IR.
1015 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1016 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1017 NewFn = Intrinsic::getOrInsertDeclaration(
1018 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1019 return true;
1020 }
1021 }
1022
1023 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1024 if (Name.starts_with("bfcvt")) {
1025 NewFn = nullptr;
1026 return true;
1027 }
1028
1029 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1030 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1031 NewFn = nullptr;
1032 return true;
1033 }
1034
1035 return false; // No other 'aarch64.neon.*'.
1036 }
1037 if (Name.consume_front("sve.")) {
1038 // 'aarch64.sve.*'.
1039 if (Name.consume_front("bf")) {
1040 if (Name == "mmla") {
1041 Type *Tys[] = {F->getReturnType(),
1042 std::next(F->arg_begin())->getType()};
1043 NewFn = Intrinsic::getOrInsertDeclaration(
1044 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1045 return true;
1046 }
1047 if (Name.consume_back(".lane")) {
1048 // 'aarch64.sve.bf*.lane'.
1049 Intrinsic::ID ID =
1050 StringSwitch<Intrinsic::ID>(Name)
1051 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1052 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1053 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1054 .Default(Intrinsic::not_intrinsic);
1055 if (ID != Intrinsic::not_intrinsic) {
1056 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1057 return true;
1058 }
1059 return false; // No other 'aarch64.sve.bf*.lane'.
1060 }
1061 return false; // No other 'aarch64.sve.bf*'.
1062 }
1063
1064 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1065 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1066 NewFn = nullptr;
1067 return true;
1068 }
1069
1070 if (Name.consume_front("convert.from.svbool")) {
1071 // 'aarch64.sve.convert.from.svbool'
1072 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1073 if (!TTy || TTy->getName() != "aarch64.svcount")
1074 return false;
1075
1076 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1077 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1078 return true;
1079 }
1080
1081 if (Name.consume_front("convert.to.svbool")) {
1082 // 'aarch64.sve.convert.to.svbool'
1083 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1084 if (!TTy || TTy->getName() != "aarch64.svcount")
1085 return false;
1086
1087 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1088 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1089 return true;
1090 }
1091
1092 if (Name.consume_front("addqv")) {
1093 // 'aarch64.sve.addqv'.
1094 if (!F->getReturnType()->isFPOrFPVectorTy())
1095 return false;
1096
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {F->getReturnType(), Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1101 return true;
1102 }
1103
1104 if (Name.consume_front("ld")) {
1105 // 'aarch64.sve.ld*'.
1106 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1107 if (LdRegex.match(Name)) {
1108 Type *ScalarTy =
1109 cast<VectorType>(F->getReturnType())->getElementType();
1110 ElementCount EC =
1111 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1112 assert(F->arg_size() == 2 &&
1113 "Expected 2 arguments for ld* intrinsic.");
1114 Type *PtrTy = F->getArg(1)->getType();
1115 Type *Ty = VectorType::get(ScalarTy, EC);
1116 static const Intrinsic::ID LoadIDs[] = {
1117 Intrinsic::aarch64_sve_ld2_sret,
1118 Intrinsic::aarch64_sve_ld3_sret,
1119 Intrinsic::aarch64_sve_ld4_sret,
1120 };
1121 NewFn = Intrinsic::getOrInsertDeclaration(
1122 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1123 return true;
1124 }
1125 return false; // No other 'aarch64.sve.ld*'.
1126 }
1127
1128 if (Name.consume_front("tuple.")) {
1129 // 'aarch64.sve.tuple.*'.
1130 if (Name.starts_with("get")) {
1131 // 'aarch64.sve.tuple.get*'.
1132 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1133 NewFn = Intrinsic::getOrInsertDeclaration(
1134 F->getParent(), Intrinsic::vector_extract, Tys);
1135 return true;
1136 }
1137
1138 if (Name.starts_with("set")) {
1139 // 'aarch64.sve.tuple.set*'.
1140 auto Args = F->getFunctionType()->params();
1141 Type *Tys[] = {Args[0], Args[2], Args[1]};
1142 NewFn = Intrinsic::getOrInsertDeclaration(
1143 F->getParent(), Intrinsic::vector_insert, Tys);
1144 return true;
1145 }
1146
1147 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1148 if (CreateTupleRegex.match(Name)) {
1149 // 'aarch64.sve.tuple.create*'.
1150 auto Args = F->getFunctionType()->params();
1151 Type *Tys[] = {F->getReturnType(), Args[1]};
1152 NewFn = Intrinsic::getOrInsertDeclaration(
1153 F->getParent(), Intrinsic::vector_insert, Tys);
1154 return true;
1155 }
1156 return false; // No other 'aarch64.sve.tuple.*'.
1157 }
1158
1159 if (Name.starts_with("rev.nxv")) {
1160 // 'aarch64.sve.rev.<Ty>'
1161 NewFn = Intrinsic::getOrInsertDeclaration(
1162 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1163 return true;
1164 }
1165
1166 return false; // No other 'aarch64.sve.*'.
1167 }
1168 if (Name.consume_front("sme.")) {
1169 // 'aarch64.sme.*'.
1170 if (Name.consume_front("ftmopa.")) {
1171 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1172 // intrinsics to model their FPMR dependency.
1173 Intrinsic::ID ID =
1174 StringSwitch<Intrinsic::ID>(Name)
1175 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1176 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1177 .Default(Intrinsic::not_intrinsic);
1178 if (ID != Intrinsic::not_intrinsic) {
1179 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1180 return true;
1181 }
1182 return false; // No other 'aarch64.sme.ftmopa.*'.
1183 }
1184
1185 return false; // No other 'aarch64.sme.*'.
1186 }
1187 }
1188 return false; // No other 'arm.*', 'aarch64.*'.
1189}
1190
1191// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1192// declarations requiring an auto-upgrade. The same set applies to the
1193// cluster (g2s) and CTA (g2s_cta) variants.
1194#define NVVM_TMA_G2S_MODES(M) \
1195 M(tile_1d, "tile.1d") \
1196 M(tile_2d, "tile.2d") \
1197 M(tile_3d, "tile.3d") \
1198 M(tile_4d, "tile.4d") \
1199 M(tile_5d, "tile.5d") \
1200 M(tile_gather4_2d, "tile.gather4.2d") \
1201 M(im2col_3d, "im2col.3d") \
1202 M(im2col_4d, "im2col.4d") \
1203 M(im2col_5d, "im2col.5d") \
1204 M(im2col_w_3d, "im2col.w.3d") \
1205 M(im2col_w_4d, "im2col.w.4d") \
1206 M(im2col_w_5d, "im2col.w.5d") \
1207 M(im2col_w_128_3d, "im2col.w.128.3d") \
1208 M(im2col_w_128_4d, "im2col.w.128.4d") \
1209 M(im2col_w_128_5d, "im2col.w.128.5d")
1210
1211// Two legacy tails are:
1212//
1213// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1215//
1216// The current tail appends a trailing i32 %flag_valid_pattern, so both
1217// legacy tails are recognized by an i1 at parameter N-2.
1218static Intrinsic::ID
1219shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1220 SmallVectorImpl<Type *> &OvlTys) {
1221 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1222 return Intrinsic::not_intrinsic;
1223
1224#define G2S_ID(ID_SUFFIX, NAME) \
1225 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1226 // clang-format off
1227 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1228 NVVM_TMA_G2S_MODES(G2S_ID)
1229 .Default(Intrinsic::not_intrinsic);
1230#undef G2S_ID
1231 // clang-format on
1232 if (ID == Intrinsic::not_intrinsic)
1233 return ID;
1234
1235 size_t NumParams = F->getFunctionType()->getNumParams();
1236
1237 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1238 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1239 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1240 return Intrinsic::not_intrinsic;
1241
1242 // The multicast mask is the parameter immediately before the i64
1243 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1244 ArrayRef<Type *> Params = F->getFunctionType()->params();
1245 size_t MaskIdx =
1246 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1247 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1248 "expected the i64 cache-hint after the multicast mask");
1249 Type *MaskTy = Params[MaskIdx];
1250 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1251 OvlTys.push_back(MaskTy);
1252
1253 return ID;
1254}
1255
1256// The legacy tail is:
1257//
1258// arg1, arg2, .. i64 %ch, i1 %flag_ch
1259//
1260// The current tail appends a trailing i32 %flag_valid_pattern, so the
1261// legacy tail is recognized by an i1 at parameter N-1.
1262static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1263 StringRef Name) {
1264 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1265 return Intrinsic::not_intrinsic;
1266
1267#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1268 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1269 // clang-format off
1270 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1271 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1272 .Default(Intrinsic::not_intrinsic);
1273#undef G2S_CTA_ID
1274 // clang-format on
1275 if (ID == Intrinsic::not_intrinsic)
1276 return ID;
1277
1278 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1279 // with i32 %flag_valid_pattern, for which N-1 is i32.
1280 if (!F->getFunctionType()
1281 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1282 ->isIntegerTy(1))
1283 return Intrinsic::not_intrinsic;
1284
1285 return ID;
1286}
1287// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1288//
1289// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1290//
1291// The current intrinsic is overloaded on the multicast-mask type and takes a
1292// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1293// parameter N-1.
1294static Intrinsic::ID
1295shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1296 SmallVectorImpl<Type *> &OvlTys) {
1297 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1298 return Intrinsic::not_intrinsic;
1299
1300 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1301 // i32 %flag_valid_pattern, for which N-1 is i32.
1302 size_t NumParams = F->getFunctionType()->getNumParams();
1303 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1304 return Intrinsic::not_intrinsic;
1305
1306 // The multicast mask is parameter 4; legacy IR only uses i16.
1307 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1308 if (!MaskTy->isIntegerTy(16))
1309 return Intrinsic::not_intrinsic;
1310 OvlTys.push_back(MaskTy);
1311
1312 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1313}
1314
1315// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1316//
1317// ..., i64 %ch, i1 %flag_ch
1318//
1319// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1320// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1321// recognized by an i1 at parameter N-1, whereas the current tail ends
1322// with an i32.
1323static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1324 StringRef Name) {
1325 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1326 return Intrinsic::not_intrinsic;
1327
1328 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1329 // i32 %flag_valid_pattern, for which N-1 is i32.
1330 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1331 return Intrinsic::not_intrinsic;
1332
1333 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1334}
1335
1336// The legacy TMA reduction intrinsics encode the reduction operator in their
1337// name, while the current ones take it as an immediate argument. Map the
1338// operator part of a legacy name to the corresponding immediate value.
1339static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1340 return StringSwitch<std::optional<unsigned>>(Name)
1341 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1342 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1343 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1344 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1345 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1346 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1347 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1348 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1349 .Default(std::nullopt);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1353 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1354 return Intrinsic::not_intrinsic;
1355
1356 auto [RedOpName, ShapeName] = Name.split('.');
1357 if (!getNVPTXTMAReductionOp(RedOpName))
1358 return Intrinsic::not_intrinsic;
1359
1360 return StringSwitch<Intrinsic::ID>(ShapeName)
1361 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1362 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1363 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1364 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1365 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1366 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1367 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1368 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1369 .Default(Intrinsic::not_intrinsic);
1370}
1371
1372static Intrinsic::ID
1373shouldUpgradeNVVMBarrierMulticastIntrinsic(Function *F, StringRef Name) {
1374 if (!Name.consume_front("mbarrier.") ||
1375 !Name.consume_back(".space.cluster") || F->arg_size() != 2)
1376 return Intrinsic::not_intrinsic;
1377
1378 Intrinsic::ID IID = F->getIntrinsicID();
1379 if (IID == Intrinsic::not_intrinsic ||
1380 Intrinsic::getType(F->getContext(), IID) == F->getFunctionType())
1381 return Intrinsic::not_intrinsic;
1382
1383 return IID;
1384}
1385
1386static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1387 StringRef Name) {
1388 if (Name.consume_front("mapa.shared.cluster"))
1389 if (F->getReturnType()->getPointerAddressSpace() ==
1390 NVPTXAS::ADDRESS_SPACE_SHARED)
1391 return Intrinsic::nvvm_mapa_shared_cluster;
1392
1393 if (Name.consume_front("cp.async.bulk.")) {
1394 Intrinsic::ID ID =
1395 StringSwitch<Intrinsic::ID>(Name)
1396 .Case("shared.cta.to.cluster",
1397 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1398 .Default(Intrinsic::not_intrinsic);
1399
1400 if (ID != Intrinsic::not_intrinsic)
1401 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1402 NVPTXAS::ADDRESS_SPACE_SHARED)
1403 return ID;
1404 }
1405
1406 return Intrinsic::not_intrinsic;
1407}
1408
1409static Intrinsic::ID
1410shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1411 if (!Name.consume_front("tcgen05.commit."))
1412 return Intrinsic::not_intrinsic;
1413
1414 if (Name.consume_front("shared."))
1415 return StringSwitch<Intrinsic::ID>(Name)
1416 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1417 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1418 .Default(Intrinsic::not_intrinsic);
1419
1420 if (Name.consume_front("mc.shared.")) {
1421 // Only upgrade older i16 mc variants.
1422 if (!F->getArg(1)->getType()->isIntegerTy(16))
1423 return Intrinsic::not_intrinsic;
1424
1425 return StringSwitch<Intrinsic::ID>(Name)
1426 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1427 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1428 .Default(Intrinsic::not_intrinsic);
1429 }
1430
1431 return Intrinsic::not_intrinsic;
1432}
1433
1434static Intrinsic::ID
1435shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name) {
1436 if (F->arg_size() != 2)
1437 return Intrinsic::not_intrinsic;
1438
1439 return StringSwitch<Intrinsic::ID>(Name)
1440 .Case("tcgen05.alloc.shared.cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1441 .Case("tcgen05.alloc.shared.cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1442 .Default(Intrinsic::not_intrinsic);
1443}
1444
1445static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1446 if (Name.consume_front("fmax."))
1447 return StringSwitch<Intrinsic::ID>(Name)
1448 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1449 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1450 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1451 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1452 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1453 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1454 .Case("ftz.nan.xorsign.abs.bf16",
1455 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1456 .Case("ftz.nan.xorsign.abs.bf16x2",
1457 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1458 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1459 .Case("ftz.xorsign.abs.bf16x2",
1460 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1461 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1462 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1463 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1464 .Case("nan.xorsign.abs.bf16x2",
1465 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1466 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1467 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1468 .Default(Intrinsic::not_intrinsic);
1469
1470 if (Name.consume_front("fmin."))
1471 return StringSwitch<Intrinsic::ID>(Name)
1472 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1473 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1474 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1475 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1476 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1477 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1478 .Case("ftz.nan.xorsign.abs.bf16",
1479 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1480 .Case("ftz.nan.xorsign.abs.bf16x2",
1481 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1482 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1483 .Case("ftz.xorsign.abs.bf16x2",
1484 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1485 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1486 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1487 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1488 .Case("nan.xorsign.abs.bf16x2",
1489 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1490 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1491 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1492 .Default(Intrinsic::not_intrinsic);
1493
1494 if (Name.consume_front("neg."))
1495 return StringSwitch<Intrinsic::ID>(Name)
1496 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1497 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1498 .Default(Intrinsic::not_intrinsic);
1499
1500 return Intrinsic::not_intrinsic;
1501}
1502
1503static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1504 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1505 FunctionType *OldFnTy = F->getFunctionType();
1506 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1507 return OldTy->getScalarType()->isIntegerTy() &&
1508 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1509 };
1510
1511 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1512 return false;
1513
1514 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1515 return false;
1516
1517 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1518 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1519 return false;
1520
1521 return true;
1522}
1523
1524// Overloaded fadd/fmul/fma intrinsic IDs, indexed by
1525// [`.ftz`][none|`.sat`|`.relu`].
1526static constexpr Intrinsic::ID NVVMFAddIIDs[2][3] = {
1527 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
1528 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1529 Intrinsic::not_intrinsic}};
1530static constexpr Intrinsic::ID NVVMFMulIIDs[2][3] = {
1531 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
1532 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1533 Intrinsic::not_intrinsic}};
1534static constexpr Intrinsic::ID NVVMFMAIIDs[2][3] = {
1535 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1536 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1537 Intrinsic::nvvm_fma_ftz_relu}};
1538
1539// Returns true if the legacy nvvm.{add,mul,fma}.<rnd>{.ftz}{.sat|.relu}.<Type>
1540// intrinsic with these modifiers existed.
1541static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID (&IIDs)[2][3],
1542 StringRef Type, bool IsRN,
1543 bool IsFTZ, bool IsSat,
1544 bool IsRelu) {
1545 const bool IsFMA = &IIDs == &NVVMFMAIIDs;
1546 const bool IsFMul = &IIDs == &NVVMFMulIIDs;
1547 return StringSwitch<bool>(Type)
1548 .Case("f", !IsRelu && !(IsSat && IsFMul))
1549 .Case("d", !IsFTZ && !IsSat && !IsRelu)
1550 .Case("f16", IsRN && (IsFMA || IsSat))
1551 .Case("v2f16", IsRN && !IsFMA && IsSat)
1552 .Case("f16x2", IsRN && IsFMA)
1553 .Cases({"bf16", "bf16x2"}, IsRN && IsFMA && !IsFTZ && !IsSat)
1554 .Default(false);
1555}
1556
1557static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1558getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][3]) {
1559 auto [Modifiers, Type] = Name.rsplit('.');
1560
1561 std::optional<llvm::RoundingMode> RoundingMode =
1562 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1563 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1564 .Case("rz", llvm::RoundingMode::TowardZero)
1565 .Case("rm", llvm::RoundingMode::TowardNegative)
1566 .Case("rp", llvm::RoundingMode::TowardPositive)
1567 .Default(std::nullopt);
1568 if (!RoundingMode)
1569 return std::nullopt;
1570
1571 StringRef Rest = Modifiers.drop_front(2);
1572 const bool IsFTZ = Rest.consume_front(".ftz");
1573 const bool IsSat = Rest.consume_front(".sat");
1574 const bool IsRelu = !IsSat && Rest.consume_front(".relu");
1575 if (!Rest.empty())
1576 return std::nullopt;
1577
1578 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1579 if (IID == Intrinsic::not_intrinsic)
1580 return std::nullopt;
1581
1582 // Reject combinations that never existed as legacy intrinsics.
1583 const bool IsRN = *RoundingMode == llvm::RoundingMode::NearestTiesToEven;
1584 if (!isValidLegacyNVVMFPArithIntrinsic(IIDs, Type, IsRN, IsFTZ, IsSat,
1585 IsRelu))
1586 return std::nullopt;
1587
1588 return std::make_pair(IID, *RoundingMode);
1589}
1590
1591static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1592 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1593 return Intrinsic::not_intrinsic;
1594
1595 return Intrinsic::nvvm_mbarrier_init;
1596}
1597
1598static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1599 return Name.consume_front("local") || Name.consume_front("shared") ||
1600 Name.consume_front("global") || Name.consume_front("constant") ||
1601 Name.consume_front("param");
1602}
1603
1604static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1605 if (!Name.consume_front("vp."))
1606 return 0;
1607 return StringSwitch<unsigned>(Name)
1608 .StartsWith("select", Instruction::Select)
1609 .StartsWith("add", Instruction::Add)
1610 .StartsWith("sub", Instruction::Sub)
1611 .StartsWith("mul", Instruction::Mul)
1612 .StartsWith("ashr", Instruction::AShr)
1613 .StartsWith("lshr", Instruction::LShr)
1614 .StartsWith("shl", Instruction::Shl)
1615 .StartsWith("or", Instruction::Or)
1616 .StartsWith("and", Instruction::And)
1617 .StartsWith("xor", Instruction::Xor)
1618 .StartsWith("fadd", Instruction::FAdd)
1619 .StartsWith("fsub", Instruction::FSub)
1620 .StartsWith("fmuladd", 0)
1621 .StartsWith("fmul", Instruction::FMul)
1622 .StartsWith("fdiv", Instruction::FDiv)
1623 .StartsWith("frem", Instruction::FRem)
1624 .StartsWith("fneg", Instruction::FNeg)
1625 .StartsWith("trunc", Instruction::Trunc)
1626 .StartsWith("zext", Instruction::ZExt)
1627 .StartsWith("sext", Instruction::SExt)
1628 .StartsWith("fptrunc", Instruction::FPTrunc)
1629 .StartsWith("fpext", Instruction::FPExt)
1630 .StartsWith("fptoui", Instruction::FPToUI)
1631 .StartsWith("fptosi", Instruction::FPToSI)
1632 .StartsWith("uitofp", Instruction::UIToFP)
1633 .StartsWith("sitofp", Instruction::SIToFP)
1634 .StartsWith("ptrtoint", Instruction::PtrToInt)
1635 .StartsWith("inttoptr", Instruction::IntToPtr)
1636 .StartsWith("icmp", Instruction::ICmp)
1637 .StartsWith("fcmp", Instruction::FCmp)
1638 .Default(0);
1639}
1640
1641static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1642 if (!Name.consume_front("vp."))
1643 return 0;
1644 return StringSwitch<Intrinsic::ID>(Name)
1645 .StartsWith("abs", Intrinsic::abs)
1646 .StartsWith("smax", Intrinsic::smax)
1647 .StartsWith("smin", Intrinsic::smin)
1648 .StartsWith("umax", Intrinsic::umax)
1649 .StartsWith("umin", Intrinsic::umin)
1650 .StartsWith("copysign", Intrinsic::copysign)
1651 .StartsWith("minnum", Intrinsic::minnum)
1652 .StartsWith("maxnum", Intrinsic::maxnum)
1653 .StartsWith("minimum", Intrinsic::minimum)
1654 .StartsWith("maximum", Intrinsic::maximum)
1655 .StartsWith("fabs", Intrinsic::fabs)
1656 .StartsWith("sqrt", Intrinsic::sqrt)
1657 .StartsWith("fma", Intrinsic::fma)
1658 .StartsWith("fmuladd", Intrinsic::fmuladd)
1659 .StartsWith("ceil", Intrinsic::ceil)
1660 .StartsWith("floor", Intrinsic::floor)
1661 .StartsWith("rint", Intrinsic::rint)
1662 .StartsWith("nearbyint", Intrinsic::nearbyint)
1663 .StartsWith("roundeven", Intrinsic::roundeven)
1664 .StartsWith("roundtozero", Intrinsic::trunc)
1665 .StartsWith("round", Intrinsic::round)
1666 .StartsWith("lrint", Intrinsic::lrint)
1667 .StartsWith("llrint", Intrinsic::llrint)
1668 .StartsWith("bitreverse", Intrinsic::bitreverse)
1669 .StartsWith("bswap", Intrinsic::bswap)
1670 .StartsWith("ctpop", Intrinsic::ctpop)
1671 .StartsWith("ctlz", Intrinsic::ctlz)
1672 .StartsWith("cttz.elts", 0)
1673 .StartsWith("cttz", Intrinsic::cttz)
1674 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1675 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1676 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1677 .StartsWith("usub.sat", Intrinsic::usub_sat)
1678 .StartsWith("fshl", Intrinsic::fshl)
1679 .StartsWith("fshr", Intrinsic::fshr)
1680 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1681 .Default(0);
1682}
1683
1687
1688static bool convertIntrinsicValidType(StringRef Name,
1689 const FunctionType *FuncTy) {
1690 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1691 if (Name.starts_with("to.fp16")) {
1692 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1693 HalfTy) &&
1694 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1695 FuncTy->getReturnType());
1696 }
1697
1698 if (Name.starts_with("from.fp16")) {
1699 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1700 HalfTy) &&
1701 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1702 FuncTy->getReturnType());
1703 }
1704
1705 return false;
1706}
1707
1708static unsigned
1709getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1710 SmallVectorImpl<Type *> &OverloadTys) {
1711 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1712 if (Defaults.empty())
1713 return 0;
1714
1715 unsigned FullArgCount = FirstDefault + Defaults.size();
1716
1717 // Only trailing default arguments can be missing.
1718 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1719 return 0;
1720
1721 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1722 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1723 NumMissingTrailingParams))
1724 return 0;
1725
1726 return FullArgCount;
1727}
1728
1729static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1730 Intrinsic::ID IID = F->getIntrinsicID();
1731 SmallVector<Type *, 4> OverloadTys;
1732
1733 unsigned FullArgCount =
1734 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1735 if (FullArgCount == 0)
1736 return false;
1737
1738 rename(F);
1739 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1740 assert(NewFn->arg_size() == FullArgCount &&
1741 "total number of default args does not match intrinsic signature");
1742 return true;
1743}
1744
1745static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1746 bool CanUpgradeDebugIntrinsicsToRecords) {
1747 assert(F && "Illegal to upgrade a non-existent Function.");
1748
1749 StringRef Name = F->getName();
1750
1751 // Quickly eliminate it, if it's not a candidate.
1752 if (!Name.consume_front("llvm.") || Name.empty())
1753 return false;
1754
1755 switch (Name[0]) {
1756 default: break;
1757 case 'a': {
1758 bool IsArm = Name.consume_front("arm.");
1759 if (IsArm || Name.consume_front("aarch64.")) {
1760 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1761 return true;
1762 break;
1763 }
1764
1765 if (Name.consume_front("amdgcn.")) {
1766 if (Name == "alignbit") {
1767 // Target specific intrinsic became redundant
1768 NewFn = Intrinsic::getOrInsertDeclaration(
1769 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1770 return true;
1771 }
1772
1773 if (Name.consume_front("atomic.")) {
1774 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1775 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1776 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1777 // and usub_sat so there's no new declaration.
1778 NewFn = nullptr;
1779 return true;
1780 }
1781 break; // No other 'amdgcn.atomic.*'
1782 }
1783
1784 if (Name.starts_with("addrspacecast.nonnull")) {
1785 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1786 // so there's no new declaration.
1787 NewFn = nullptr;
1788 return true;
1789 }
1790
1791 switch (F->getIntrinsicID()) {
1792 default:
1793 break;
1794 // Legacy asyncmark intrinsics without the stage operand.
1795 case Intrinsic::amdgcn_asyncmark:
1796 case Intrinsic::amdgcn_wait_asyncmark: {
1797 Intrinsic::ID ID = F->getIntrinsicID();
1798 if (F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
1799 rename(F);
1800 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1801 return true;
1802 }
1803 break;
1804 }
1805 // Legacy wmma iu intrinsics without the optional clamp operand.
1806 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1807 if (F->arg_size() == 7) {
1808 NewFn = nullptr;
1809 return true;
1810 }
1811 break;
1812 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1813 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1814 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1815 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1816 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1817 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1818 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1819 if (F->arg_size() == 8) {
1820 NewFn = nullptr;
1821 return true;
1822 }
1823 break;
1824 }
1825
1826 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1827 Name.consume_front("flat.atomic.")) {
1828 if (Name.starts_with("fadd") ||
1829 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1830 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1831 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1832 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1833 // declaration.
1834 NewFn = nullptr;
1835 return true;
1836 }
1837 }
1838
1839 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1840 NewFn = nullptr;
1841 return true;
1842 }
1843
1844 if (Name.starts_with("ldexp.")) {
1845 // Target specific intrinsic became redundant
1846 NewFn = Intrinsic::getOrInsertDeclaration(
1847 F->getParent(), Intrinsic::ldexp,
1848 {F->getReturnType(), F->getArg(1)->getType()});
1849 return true;
1850 }
1851 break; // No other 'amdgcn.*'
1852 }
1853
1854 break;
1855 }
1856 case 'c': {
1857 if (F->arg_size() == 1) {
1858 if (Name.consume_front("convert.")) {
1859 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1860 NewFn = nullptr;
1861 return true;
1862 }
1863 }
1864
1865 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1866 .StartsWith("ctlz.", Intrinsic::ctlz)
1867 .StartsWith("cttz.", Intrinsic::cttz)
1868 .Default(Intrinsic::not_intrinsic);
1869 if (ID != Intrinsic::not_intrinsic) {
1870 rename(F);
1871 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1872 F->arg_begin()->getType());
1873 return true;
1874 }
1875 }
1876
1877 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1878 if (Name == "coro.end" &&
1879 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1880 CoroEndID = Intrinsic::coro_end;
1881 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1882 CoroEndID = Intrinsic::coro_end_async;
1883
1884 if (CoroEndID != Intrinsic::not_intrinsic) {
1885 rename(F);
1886 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1887 return true;
1888 }
1889
1890 break;
1891 }
1892 case 'd':
1893 if (Name.consume_front("dbg.")) {
1894 // Mark debug intrinsics for upgrade to new debug format.
1895 if (CanUpgradeDebugIntrinsicsToRecords) {
1896 if (Name == "addr" || Name == "value" || Name == "assign" ||
1897 Name == "declare" || Name == "label") {
1898 // There's no function to replace these with.
1899 NewFn = nullptr;
1900 // But we do want these to get upgraded.
1901 return true;
1902 }
1903 }
1904 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1905 // converted to DbgVariableRecords later.
1906 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1907 rename(F);
1908 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1909 Intrinsic::dbg_value);
1910 return true;
1911 }
1912 break; // No other 'dbg.*'.
1913 }
1914 break;
1915 case 'e':
1916 if (Name.consume_front("experimental.vector.")) {
1917 Intrinsic::ID ID =
1918 StringSwitch<Intrinsic::ID>(Name)
1919 // Skip over extract.last.active, otherwise it will be 'upgraded'
1920 // to a regular vector extract which is a different operation.
1921 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1922 .StartsWith("extract.", Intrinsic::vector_extract)
1923 .StartsWith("insert.", Intrinsic::vector_insert)
1924 .StartsWith("reverse.", Intrinsic::vector_reverse)
1925 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1926 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1927 .StartsWith("partial.reduce.add",
1928 Intrinsic::vector_partial_reduce_add)
1929 .Default(Intrinsic::not_intrinsic);
1930 if (ID != Intrinsic::not_intrinsic) {
1931 const auto *FT = F->getFunctionType();
1932 SmallVector<Type *, 2> Tys;
1933 if (ID == Intrinsic::vector_extract ||
1934 ID == Intrinsic::vector_interleave2)
1935 // Extracting overloads the return type.
1936 Tys.push_back(FT->getReturnType());
1937 if (ID != Intrinsic::vector_interleave2)
1938 Tys.push_back(FT->getParamType(0));
1939 if (ID == Intrinsic::vector_insert ||
1940 ID == Intrinsic::vector_partial_reduce_add)
1941 // Inserting overloads the inserted type.
1942 Tys.push_back(FT->getParamType(1));
1943 rename(F);
1944 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1945 return true;
1946 }
1947
1948 if (Name.consume_front("reduce.")) {
1949 SmallVector<StringRef, 2> Groups;
1950 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1951 if (R.match(Name, &Groups))
1952 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1953 .Case("add", Intrinsic::vector_reduce_add)
1954 .Case("mul", Intrinsic::vector_reduce_mul)
1955 .Case("and", Intrinsic::vector_reduce_and)
1956 .Case("or", Intrinsic::vector_reduce_or)
1957 .Case("xor", Intrinsic::vector_reduce_xor)
1958 .Case("smax", Intrinsic::vector_reduce_smax)
1959 .Case("smin", Intrinsic::vector_reduce_smin)
1960 .Case("umax", Intrinsic::vector_reduce_umax)
1961 .Case("umin", Intrinsic::vector_reduce_umin)
1962 .Case("fmax", Intrinsic::vector_reduce_fmax)
1963 .Case("fmin", Intrinsic::vector_reduce_fmin)
1964 .Default(Intrinsic::not_intrinsic);
1965
1966 bool V2 = false;
1967 if (ID == Intrinsic::not_intrinsic) {
1968 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1969 Groups.clear();
1970 V2 = true;
1971 if (R2.match(Name, &Groups))
1972 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1973 .Case("fadd", Intrinsic::vector_reduce_fadd)
1974 .Case("fmul", Intrinsic::vector_reduce_fmul)
1975 .Default(Intrinsic::not_intrinsic);
1976 }
1977 if (ID != Intrinsic::not_intrinsic) {
1978 rename(F);
1979 auto Args = F->getFunctionType()->params();
1980 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1981 {Args[V2 ? 1 : 0]});
1982 return true;
1983 }
1984 break; // No other 'expermental.vector.reduce.*'.
1985 }
1986
1987 if (Name.consume_front("splice"))
1988 return true;
1989 break; // No other 'experimental.vector.*'.
1990 }
1991 if (Name.consume_front("experimental.stepvector.")) {
1992 Intrinsic::ID ID = Intrinsic::stepvector;
1993 rename(F);
1994 NewFn = Intrinsic::getOrInsertDeclaration(
1995 F->getParent(), ID, F->getFunctionType()->getReturnType());
1996 return true;
1997 }
1998 break; // No other 'e*'.
1999 case 'f':
2000 if (Name.starts_with("flt.rounds")) {
2001 rename(F);
2002 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2003 Intrinsic::get_rounding);
2004 return true;
2005 }
2006 break;
2007 case 'i':
2008 if (Name.starts_with("invariant.group.barrier")) {
2009 // Rename invariant.group.barrier to launder.invariant.group
2010 auto Args = F->getFunctionType()->params();
2011 Type* ObjectPtr[1] = {Args[0]};
2012 rename(F);
2013 NewFn = Intrinsic::getOrInsertDeclaration(
2014 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
2015 return true;
2016 }
2017 break;
2018 case 'l': {
2019 bool IsLifetimeStart = Name.consume_front("lifetime.start");
2020 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
2021 if (IsLifetimeStart || IsLifetimeEnd) {
2022 if (F->arg_size() == 2) {
2023 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2024 : Intrinsic::lifetime_end;
2025 rename(F);
2026 // Old 2 argument form of these intrinsics have [Size, Ptr] as
2027 // arguments. Use the Ptr argument to create new declaration.
2028 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2029 F->getArg(1)->getType());
2030 return true;
2031 } else if (F->arg_size() == 1 && Name == ".i64") {
2032 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
2033 // Autoupgrade prior to
2034 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
2035 // intrinsic with no expected calls. To allow auto-upgrade process to
2036 // delete such invalid intrinsic declaration, set NewFn = nullptr
2037 // and return true here. If there are actual calls to this intrinsic
2038 // (which is not expected), they will be deleted in
2039 // UpgradeIntrinsicCall.
2040 NewFn = nullptr;
2041 return true;
2042 }
2043 }
2044 break;
2045 }
2046 case 'm': {
2047 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2048 // alignment parameter to embedding the alignment as an attribute of
2049 // the pointer args.
2050 if (unsigned ID = StringSwitch<unsigned>(Name)
2051 .StartsWith("memcpy.", Intrinsic::memcpy)
2052 .StartsWith("memmove.", Intrinsic::memmove)
2053 .Default(0)) {
2054 if (F->arg_size() == 5) {
2055 rename(F);
2056 // Get the types of dest, src, and len
2057 ArrayRef<Type *> ParamTypes =
2058 F->getFunctionType()->params().slice(0, 3);
2059 NewFn =
2060 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2061 return true;
2062 }
2063 }
2064 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2065 rename(F);
2066 // Get the types of dest, and len
2067 const auto *FT = F->getFunctionType();
2068 Type *ParamTypes[2] = {
2069 FT->getParamType(0), // Dest
2070 FT->getParamType(2) // len
2071 };
2072 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2073 Intrinsic::memset, ParamTypes);
2074 return true;
2075 }
2076
2077 unsigned MaskedID =
2078 StringSwitch<unsigned>(Name)
2079 .StartsWith("masked.load", Intrinsic::masked_load)
2080 .StartsWith("masked.gather", Intrinsic::masked_gather)
2081 .StartsWith("masked.store", Intrinsic::masked_store)
2082 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2083 .Default(0);
2084 if (MaskedID && F->arg_size() == 4) {
2085 rename(F);
2086 if (MaskedID == Intrinsic::masked_load ||
2087 MaskedID == Intrinsic::masked_gather) {
2088 NewFn = Intrinsic::getOrInsertDeclaration(
2089 F->getParent(), MaskedID,
2090 {F->getReturnType(), F->getArg(0)->getType()});
2091 return true;
2092 }
2093 NewFn = Intrinsic::getOrInsertDeclaration(
2094 F->getParent(), MaskedID,
2095 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2096 return true;
2097 }
2098 break;
2099 }
2100 case 'n': {
2101 if (Name.consume_front("nvvm.")) {
2102 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2103 if (F->arg_size() == 1) {
2104 Intrinsic::ID IID =
2105 StringSwitch<Intrinsic::ID>(Name)
2106 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2107 .Case("clz.i", Intrinsic::ctlz)
2108 .Case("popc.i", Intrinsic::ctpop)
2109 .Default(Intrinsic::not_intrinsic);
2110 if (IID != Intrinsic::not_intrinsic) {
2111 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2112 {F->getReturnType()});
2113 return true;
2114 }
2115 } else if (F->arg_size() == 2) {
2116 Intrinsic::ID IID =
2117 StringSwitch<Intrinsic::ID>(Name)
2118 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2119 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2120 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2121 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2122 .Cases({"mulhi.s", "mulhi.i", "mulhi.ll"}, Intrinsic::smulh)
2123 .Cases({"mulhi.us", "mulhi.ui", "mulhi.ull"}, Intrinsic::umulh)
2124 .Default(Intrinsic::not_intrinsic);
2125 if (IID != Intrinsic::not_intrinsic) {
2126 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2127 {F->getReturnType()});
2128 return true;
2129 }
2130 }
2131
2132 // Check for nvvm intrinsics that need a return type adjustment.
2133 {
2134 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2135 if (IID != Intrinsic::not_intrinsic &&
2136 isLegacyNVPTXBF16IntSignature(F, IID)) {
2137 NewFn = nullptr;
2138 return true;
2139 }
2140 }
2141
2142 // Upgrade Distributed Shared Memory Intrinsics
2143 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2144 if (IID != Intrinsic::not_intrinsic) {
2145 rename(F);
2146 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2147 return true;
2148 }
2149
2150 // Upgrade TMA reduction intrinsics
2151 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2152 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2153 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2154 if (IID != Intrinsic::not_intrinsic) {
2155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2156 return true;
2157 }
2158
2159 // Upgrade the shared::cluster mbarrier expect_tx/complete_tx/arrive*
2160 // intrinsics to the forms carrying the 32-bit multicast CTA mask and
2161 // the is-multicast flag.
2162 IID = shouldUpgradeNVVMBarrierMulticastIntrinsic(F, Name);
2163 if (IID != Intrinsic::not_intrinsic) {
2164 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2165 return true;
2166 }
2167
2168 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2169 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2170 if (IID != Intrinsic::not_intrinsic) {
2171 rename(F);
2172 NewFn = Intrinsic::getOrInsertDeclaration(
2173 F->getParent(), IID, F->getReturnType(),
2174 F->getFunctionType()->params());
2175 return true;
2176 }
2177
2178 // Upgrade tcgen05.alloc.shared.* to anyptr alloc intrinsic.
2179 IID = shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(F, Name);
2180 if (IID != Intrinsic::not_intrinsic) {
2181 rename(F);
2182 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2183 {F->getArg(0)->getType()});
2184 return true;
2185 }
2186
2187 // Upgrade TMA copy G2S CTA intrinsics.
2188 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2189 if (IID != Intrinsic::not_intrinsic) {
2190 rename(F);
2191 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2192 return true;
2193 }
2194
2195 // Upgrade TMA copy G2S (cluster) intrinsics.
2196 SmallVector<Type *, 1> OvlTys;
2197 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2198 if (IID != Intrinsic::not_intrinsic) {
2199 rename(F);
2200 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2201 return true;
2202 }
2203
2204 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2205 // (multicast-mask overloading + trailing flag_valid_pattern).
2206 SmallVector<Type *, 1> BulkG2SOvlTys;
2207 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2208 if (IID != Intrinsic::not_intrinsic) {
2209 rename(F);
2210 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2211 BulkG2SOvlTys);
2212 return true;
2213 }
2214
2215 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2216 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2217 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2218 if (IID != Intrinsic::not_intrinsic) {
2219 rename(F);
2220 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2221 return true;
2222 }
2223
2224 // Upgrade mbarrier.init intrinsics missing the layout operand.
2225 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2226 if (IID != Intrinsic::not_intrinsic) {
2227 rename(F);
2228 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2229 F->getArg(0)->getType());
2230 return true;
2231 }
2232
2233 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2234 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2235 //
2236 // TODO: We could add lohi.i2d.
2237 bool Expand = false;
2238 if (Name.consume_front("abs."))
2239 // nvvm.abs.{i,ii}
2240 Expand =
2241 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2242 else if (Name.consume_front("fabs."))
2243 // nvvm.fabs.{f,ftz.f,d}
2244 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2245 else if (Name.consume_front("add."))
2246 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2247 Expand = getNVVMFPArithUpgrade(Name, NVVMFAddIIDs).has_value();
2248 else if (Name.consume_front("mul."))
2249 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2250 Expand = getNVVMFPArithUpgrade(Name, NVVMFMulIIDs).has_value();
2251 else if (Name.consume_front("ex2.approx."))
2252 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2253 Expand =
2254 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2255 else if (Name.consume_front("fma."))
2256 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
2257 Expand = getNVVMFPArithUpgrade(Name, NVVMFMAIIDs).has_value();
2258 else if (Name.consume_front("atomic.load."))
2259 // nvvm.atomic.load.add.{f32,f64}.p
2260 // nvvm.atomic.load.{inc,dec}.32.p
2261 Expand = StringSwitch<bool>(Name)
2262 .StartsWith("add.f32.p", true)
2263 .StartsWith("add.f64.p", true)
2264 .StartsWith("inc.32.p", true)
2265 .StartsWith("dec.32.p", true)
2266 .Default(false);
2267 else if (Name.consume_front("atomic."))
2268 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2269 // nvvm.atomic.cas.gen.i.{cta,sys}
2270 Expand = StringSwitch<bool>(Name)
2271 .StartsWith("add.gen.", true)
2272 .StartsWith("exch.gen.", true)
2273 .StartsWith("max.gen.", true)
2274 .StartsWith("min.gen.", true)
2275 .StartsWith("inc.gen.", true)
2276 .StartsWith("dec.gen.", true)
2277 .StartsWith("and.gen.", true)
2278 .StartsWith("or.gen.", true)
2279 .StartsWith("xor.gen.", true)
2280 .StartsWith("cas.gen.", true)
2281 .Default(false);
2282 else if (Name.consume_front("bitcast."))
2283 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2284 Expand =
2285 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2286 else if (Name.consume_front("rotate."))
2287 // nvvm.rotate.{b32,b64,right.b64}
2288 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2289 else if (Name.consume_front("ptr.gen.to."))
2290 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2291 Expand = consumeNVVMPtrAddrSpace(Name);
2292 else if (Name.consume_front("ptr."))
2293 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2294 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2295 else if (Name.consume_front("ldg.global."))
2296 // nvvm.ldg.global.{i,p,f}
2297 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2298 Name.starts_with("p."));
2299 else
2300 Expand = StringSwitch<bool>(Name)
2301 .Case("barrier0", true)
2302 .Case("barrier.n", true)
2303 .Case("barrier.sync.cnt", true)
2304 .Case("barrier.sync", true)
2305 .Case("barrier", true)
2306 .Case("bar.sync", true)
2307 .Case("barrier0.popc", true)
2308 .Case("barrier0.and", true)
2309 .Case("barrier0.or", true)
2310 .Case("clz.ll", true)
2311 .Case("popc.ll", true)
2312 .Case("h2f", true)
2313 .Case("swap.lo.hi.b64", true)
2314 .Case("tanh.approx.f32", true)
2315 .Default(false);
2316
2317 if (Expand) {
2318 NewFn = nullptr;
2319 return true;
2320 }
2321 break; // No other 'nvvm.*'.
2322 }
2323 break;
2324 }
2325 case 'o':
2326 if (Name.starts_with("objectsize.")) {
2327 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2328 if (F->arg_size() == 2 || F->arg_size() == 3) {
2329 rename(F);
2330 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2331 Intrinsic::objectsize, Tys);
2332 return true;
2333 }
2334 }
2335 break;
2336
2337 case 'p':
2338 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2339 rename(F);
2340 NewFn = Intrinsic::getOrInsertDeclaration(
2341 F->getParent(), Intrinsic::ptr_annotation,
2342 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2343 return true;
2344 }
2345 break;
2346
2347 case 'r': {
2348 if (Name.consume_front("riscv.")) {
2349 Intrinsic::ID ID;
2350 ID = StringSwitch<Intrinsic::ID>(Name)
2351 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2352 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2353 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2354 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2355 .Default(Intrinsic::not_intrinsic);
2356 if (ID != Intrinsic::not_intrinsic) {
2357 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2358 rename(F);
2359 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2360 return true;
2361 }
2362 break; // No other applicable upgrades.
2363 }
2364
2365 ID = StringSwitch<Intrinsic::ID>(Name)
2366 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2367 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2368 .Default(Intrinsic::not_intrinsic);
2369 if (ID != Intrinsic::not_intrinsic) {
2370 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2371 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2372 rename(F);
2373 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2374 return true;
2375 }
2376 break; // No other applicable upgrades.
2377 }
2378
2379 ID = StringSwitch<Intrinsic::ID>(Name)
2380 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2381 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2382 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2383 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2384 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2385 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2386 .Default(Intrinsic::not_intrinsic);
2387 if (ID != Intrinsic::not_intrinsic) {
2388 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2389 rename(F);
2390 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2391 return true;
2392 }
2393 break; // No other applicable upgrades.
2394 }
2395
2396 // Replace llvm.riscv.clmul with llvm.clmul.
2397 if (Name == "clmul.i32" || Name == "clmul.i64") {
2398 NewFn = Intrinsic::getOrInsertDeclaration(
2399 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2400 return true;
2401 }
2402
2403 break; // No other 'riscv.*' intrinsics
2404 }
2405 } break;
2406
2407 case 's':
2408 if (Name == "stackprotectorcheck") {
2409 NewFn = nullptr;
2410 return true;
2411 }
2412 if (Name.starts_with("strip.invariant.group")) {
2413 // For clang's usage it would be safe to just drop the
2414 // strip.invariant.group, but to be conservative replace with the
2415 // stronger launder.invariant.group instead.
2416 NewFn = Intrinsic::getOrInsertDeclaration(
2417 F->getParent(), Intrinsic::launder_invariant_group,
2418 F->getReturnType());
2419 return true;
2420 }
2421 break;
2422
2423 case 't':
2424 if (Name == "thread.pointer") {
2425 NewFn = Intrinsic::getOrInsertDeclaration(
2426 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2427 return true;
2428 }
2429 break;
2430
2431 case 'v': {
2432 if (Name == "var.annotation" && F->arg_size() == 4) {
2433 rename(F);
2434 NewFn = Intrinsic::getOrInsertDeclaration(
2435 F->getParent(), Intrinsic::var_annotation,
2436 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2437 return true;
2438 }
2439 if (Name.consume_front("vector.splice")) {
2440 if (Name.starts_with(".left") || Name.starts_with(".right"))
2441 break;
2442 return true;
2443 }
2444 if (shouldUpgradeVPIntrinsic(Name))
2445 return true;
2446 break;
2447 }
2448
2449 case 'w':
2450 if (Name.consume_front("wasm.")) {
2451 Intrinsic::ID ID =
2452 StringSwitch<Intrinsic::ID>(Name)
2453 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2454 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2455 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2456 .Default(Intrinsic::not_intrinsic);
2457 if (ID != Intrinsic::not_intrinsic) {
2458 rename(F);
2459 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2460 F->getReturnType());
2461 return true;
2462 }
2463
2464 if (Name.consume_front("dot.i8x16.i7x16.")) {
2465 ID = StringSwitch<Intrinsic::ID>(Name)
2466 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2467 .Case("add.signed",
2468 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2469 .Default(Intrinsic::not_intrinsic);
2470 if (ID != Intrinsic::not_intrinsic) {
2471 rename(F);
2472 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2473 return true;
2474 }
2475 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2476 }
2477 break; // No other 'wasm.*'.
2478 }
2479 break;
2480
2481 case 'x':
2482 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2483 return true;
2484 }
2485
2486 auto *ST = dyn_cast<StructType>(F->getReturnType());
2487 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2488 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2489 // Replace return type with literal non-packed struct. Only do this for
2490 // intrinsics declared to return a struct, not for intrinsics with
2491 // overloaded return type, in which case the exact struct type will be
2492 // mangled into the name.
2493 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2494 FunctionType *FT = F->getFunctionType();
2495 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2496 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2497 std::string Name = F->getName().str();
2498 rename(F);
2499 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2500 Name, F->getParent());
2501
2502 // The new function may also need remangling.
2503 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2504 NewFn = *Result;
2505 return true;
2506 }
2507 }
2508
2509 // Remangle our intrinsic since we upgrade the mangling
2510 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2511 if (Result != std::nullopt) {
2512 NewFn = *Result;
2513 return true;
2514 }
2515
2516 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2517 return true;
2518
2519 // This may not belong here. This function is effectively being overloaded
2520 // to both detect an intrinsic which needs upgrading, and to provide the
2521 // upgraded form of the intrinsic. We should perhaps have two separate
2522 // functions for this.
2523
2524 return false;
2525}
2526
2527bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2528 bool CanUpgradeDebugIntrinsicsToRecords) {
2529 NewFn = nullptr;
2530 bool Upgraded =
2531 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2532
2533 // Upgrade intrinsic attributes. This does not change the function.
2534 if (NewFn)
2535 F = NewFn;
2536 if (Intrinsic::ID id = F->getIntrinsicID()) {
2537 // Only do this if the intrinsic signature is valid.
2538 SmallVector<Type *> OverloadTys;
2539 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2540 F->setAttributes(
2541 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2542 }
2543 return Upgraded;
2544}
2545
2546GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2547 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2548 GV->getName() == "llvm.global_dtors")) ||
2549 !GV->hasInitializer())
2550 return nullptr;
2551 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2552 if (!ATy)
2553 return nullptr;
2554 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2555 if (!STy || STy->getNumElements() != 2)
2556 return nullptr;
2557
2558 IRBuilder<> IRB(*GV->getParent());
2559 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2560 IRB.getPtrTy());
2561 Constant *Init = GV->getInitializer();
2562 unsigned N = Init->getNumOperands();
2563 std::vector<Constant *> NewCtors(N);
2564 for (unsigned i = 0; i != N; ++i) {
2565 auto Ctor = cast<Constant>(Init->getOperand(i));
2566 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2567 Ctor->getAggregateElement(1),
2568 ConstantPointerNull::get(IRB.getPtrTy()));
2569 }
2570 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2571
2572 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2573 NewInit, GV->getName());
2574}
2575
2576// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2577// to byte shuffles.
2578static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2579 unsigned Shift) {
2580 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2581 unsigned NumElts = ResultTy->getNumElements() * 8;
2582
2583 // Bitcast from a 64-bit element type to a byte element type.
2584 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2585 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2586
2587 // We'll be shuffling in zeroes.
2588 Value *Res = Constant::getNullValue(VecTy);
2589
2590 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2591 // we'll just return the zero vector.
2592 if (Shift < 16) {
2593 int Idxs[64];
2594 // 256/512-bit version is split into 2/4 16-byte lanes.
2595 for (unsigned l = 0; l != NumElts; l += 16)
2596 for (unsigned i = 0; i != 16; ++i) {
2597 unsigned Idx = NumElts + i - Shift;
2598 if (Idx < NumElts)
2599 Idx -= NumElts - 16; // end of lane, switch operand.
2600 Idxs[l + i] = Idx + l;
2601 }
2602
2603 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2604 }
2605
2606 // Bitcast back to a 64-bit element type.
2607 return Builder.CreateBitCast(Res, ResultTy, "cast");
2608}
2609
2610// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2611// to byte shuffles.
2612static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2613 unsigned Shift) {
2614 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2615 unsigned NumElts = ResultTy->getNumElements() * 8;
2616
2617 // Bitcast from a 64-bit element type to a byte element type.
2618 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2619 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2620
2621 // We'll be shuffling in zeroes.
2622 Value *Res = Constant::getNullValue(VecTy);
2623
2624 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2625 // we'll just return the zero vector.
2626 if (Shift < 16) {
2627 int Idxs[64];
2628 // 256/512-bit version is split into 2/4 16-byte lanes.
2629 for (unsigned l = 0; l != NumElts; l += 16)
2630 for (unsigned i = 0; i != 16; ++i) {
2631 unsigned Idx = i + Shift;
2632 if (Idx >= 16)
2633 Idx += NumElts - 16; // end of lane, switch operand.
2634 Idxs[l + i] = Idx + l;
2635 }
2636
2637 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2638 }
2639
2640 // Bitcast back to a 64-bit element type.
2641 return Builder.CreateBitCast(Res, ResultTy, "cast");
2642}
2643
2644static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2645 unsigned NumElts) {
2646 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2647 llvm::VectorType *MaskTy = FixedVectorType::get(
2648 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2649 Mask = Builder.CreateBitCast(Mask, MaskTy);
2650
2651 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2652 // i8 and we need to extract down to the right number of elements.
2653 if (NumElts <= 4) {
2654 int Indices[4];
2655 for (unsigned i = 0; i != NumElts; ++i)
2656 Indices[i] = i;
2657 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2658 "extract");
2659 }
2660
2661 return Mask;
2662}
2663
2664static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2665 Value *Op1) {
2666 // If the mask is all ones just emit the first operation.
2667 if (const auto *C = dyn_cast<Constant>(Mask))
2668 if (C->isAllOnesValue())
2669 return Op0;
2670
2671 Mask = getX86MaskVec(Builder, Mask,
2672 cast<FixedVectorType>(Op0->getType())->getNumElements());
2673 return Builder.CreateSelect(Mask, Op0, Op1);
2674}
2675
2676static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2677 Value *Op1) {
2678 // If the mask is all ones just emit the first operation.
2679 if (const auto *C = dyn_cast<Constant>(Mask))
2680 if (C->isAllOnesValue())
2681 return Op0;
2682
2683 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2684 Mask->getType()->getIntegerBitWidth());
2685 Mask = Builder.CreateBitCast(Mask, MaskTy);
2686 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2687 return Builder.CreateSelect(Mask, Op0, Op1);
2688}
2689
2690// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2691// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2692// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2693static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2694 Value *Op1, Value *Shift,
2695 Value *Passthru, Value *Mask,
2696 bool IsVALIGN) {
2697 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2698
2699 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2700 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2701 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2702 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2703
2704 // Mask the immediate for VALIGN.
2705 if (IsVALIGN)
2706 ShiftVal &= (NumElts - 1);
2707
2708 // If palignr is shifting the pair of vectors more than the size of two
2709 // lanes, emit zero.
2710 if (ShiftVal >= 32)
2711 return llvm::Constant::getNullValue(Op0->getType());
2712
2713 // If palignr is shifting the pair of input vectors more than one lane,
2714 // but less than two lanes, convert to shifting in zeroes.
2715 if (ShiftVal > 16) {
2716 ShiftVal -= 16;
2717 Op1 = Op0;
2718 Op0 = llvm::Constant::getNullValue(Op0->getType());
2719 }
2720
2721 int Indices[64];
2722 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2723 for (unsigned l = 0; l < NumElts; l += 16) {
2724 for (unsigned i = 0; i != 16; ++i) {
2725 unsigned Idx = ShiftVal + i;
2726 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2727 Idx += NumElts - 16; // End of lane, switch operand.
2728 Indices[l + i] = Idx + l;
2729 }
2730 }
2731
2732 Value *Align = Builder.CreateShuffleVector(
2733 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2734
2735 return emitX86Select(Builder, Mask, Align, Passthru);
2736}
2737
2738static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2739 bool ZeroMask, bool IndexForm) {
2740 Type *Ty = CI.getType();
2741 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2742 unsigned EltWidth = Ty->getScalarSizeInBits();
2743 bool IsFloat = Ty->isFPOrFPVectorTy();
2744 Intrinsic::ID IID;
2745 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2746 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2747 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2748 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2749 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2750 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2751 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2752 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2753 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2754 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2755 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2756 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2757 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2758 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2759 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2760 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2761 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2762 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2763 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2764 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2765 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2766 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2767 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2768 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2769 else if (VecWidth == 128 && EltWidth == 16)
2770 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2771 else if (VecWidth == 256 && EltWidth == 16)
2772 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2773 else if (VecWidth == 512 && EltWidth == 16)
2774 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2775 else if (VecWidth == 128 && EltWidth == 8)
2776 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2777 else if (VecWidth == 256 && EltWidth == 8)
2778 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2779 else if (VecWidth == 512 && EltWidth == 8)
2780 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2781 else
2782 llvm_unreachable("Unexpected intrinsic");
2783
2784 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2785 CI.getArgOperand(2) };
2786
2787 // If this isn't index form we need to swap operand 0 and 1.
2788 if (!IndexForm)
2789 std::swap(Args[0], Args[1]);
2790
2791 Value *V = Builder.CreateIntrinsic(IID, Args);
2792 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2793 : Builder.CreateBitCast(CI.getArgOperand(1),
2794 Ty);
2795 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2796}
2797
2798static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2799 Intrinsic::ID IID) {
2800 Type *Ty = CI.getType();
2801 Value *Op0 = CI.getOperand(0);
2802 Value *Op1 = CI.getOperand(1);
2803 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2804
2805 if (CI.arg_size() == 4) { // For masked intrinsics.
2806 Value *VecSrc = CI.getOperand(2);
2807 Value *Mask = CI.getOperand(3);
2808 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2809 }
2810 return Res;
2811}
2812
2813static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2814 bool IsRotateRight) {
2815 Type *Ty = CI.getType();
2816 Value *Src = CI.getArgOperand(0);
2817 Value *Amt = CI.getArgOperand(1);
2818
2819 // Amount may be scalar immediate, in which case create a splat vector.
2820 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2821 // we only care about the lowest log2 bits anyway.
2822 if (Amt->getType() != Ty) {
2823 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2824 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2825 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2826 }
2827
2828 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2829 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2830
2831 if (CI.arg_size() == 4) { // For masked intrinsics.
2832 Value *VecSrc = CI.getOperand(2);
2833 Value *Mask = CI.getOperand(3);
2834 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2835 }
2836 return Res;
2837}
2838
2839static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2840 bool IsSigned) {
2841 Type *Ty = CI.getType();
2842 Value *LHS = CI.getArgOperand(0);
2843 Value *RHS = CI.getArgOperand(1);
2844
2845 CmpInst::Predicate Pred;
2846 switch (Imm) {
2847 case 0x0:
2848 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2849 break;
2850 case 0x1:
2851 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2852 break;
2853 case 0x2:
2854 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2855 break;
2856 case 0x3:
2857 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2858 break;
2859 case 0x4:
2860 Pred = ICmpInst::ICMP_EQ;
2861 break;
2862 case 0x5:
2863 Pred = ICmpInst::ICMP_NE;
2864 break;
2865 case 0x6:
2866 return Constant::getNullValue(Ty); // FALSE
2867 case 0x7:
2868 return Constant::getAllOnesValue(Ty); // TRUE
2869 default:
2870 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2871 }
2872
2873 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2874 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2875 return Ext;
2876}
2877
2878static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2879 bool IsShiftRight, bool ZeroMask) {
2880 Type *Ty = CI.getType();
2881 Value *Op0 = CI.getArgOperand(0);
2882 Value *Op1 = CI.getArgOperand(1);
2883 Value *Amt = CI.getArgOperand(2);
2884
2885 if (IsShiftRight)
2886 std::swap(Op0, Op1);
2887
2888 // Amount may be scalar immediate, in which case create a splat vector.
2889 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2890 // we only care about the lowest log2 bits anyway.
2891 if (Amt->getType() != Ty) {
2892 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2893 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2894 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2895 }
2896
2897 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2898 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2899
2900 unsigned NumArgs = CI.arg_size();
2901 if (NumArgs >= 4) { // For masked intrinsics.
2902 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2903 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2904 CI.getArgOperand(0);
2905 Value *Mask = CI.getOperand(NumArgs - 1);
2906 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2907 }
2908 return Res;
2909}
2910
2911static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2912 Value *Mask, bool Aligned) {
2913 const Align Alignment =
2914 Aligned
2915 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2916 : Align(1);
2917
2918 // If the mask is all ones just emit a regular store.
2919 if (const auto *C = dyn_cast<Constant>(Mask))
2920 if (C->isAllOnesValue())
2921 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2922
2923 // Convert the mask from an integer type to a vector of i1.
2924 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2925 Mask = getX86MaskVec(Builder, Mask, NumElts);
2926 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2927}
2928
2929static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2930 Value *Passthru, Value *Mask, bool Aligned) {
2931 Type *ValTy = Passthru->getType();
2932 const Align Alignment =
2933 Aligned
2934 ? Align(
2935 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2936 8)
2937 : Align(1);
2938
2939 // If the mask is all ones just emit a regular store.
2940 if (const auto *C = dyn_cast<Constant>(Mask))
2941 if (C->isAllOnesValue())
2942 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2943
2944 // Convert the mask from an integer type to a vector of i1.
2945 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2946 Mask = getX86MaskVec(Builder, Mask, NumElts);
2947 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2948}
2949
2950static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2951 Type *Ty = CI.getType();
2952 Value *Op0 = CI.getArgOperand(0);
2953 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2954 {Op0, Builder.getInt1(false)});
2955 if (CI.arg_size() == 3)
2956 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2957 return Res;
2958}
2959
2960static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2961 Type *Ty = CI.getType();
2962
2963 // Arguments have a vXi32 type so cast to vXi64.
2964 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2965 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2966
2967 if (IsSigned) {
2968 // Shift left then arithmetic shift right.
2969 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2970 LHS = Builder.CreateShl(LHS, ShiftAmt);
2971 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2972 RHS = Builder.CreateShl(RHS, ShiftAmt);
2973 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2974 } else {
2975 // Clear the upper bits.
2976 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2977 LHS = Builder.CreateAnd(LHS, Mask);
2978 RHS = Builder.CreateAnd(RHS, Mask);
2979 }
2980
2981 Value *Res = Builder.CreateMul(LHS, RHS);
2982
2983 if (CI.arg_size() == 4)
2984 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2985
2986 return Res;
2987}
2988
2989// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2990static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2991 Value *Mask) {
2992 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2993 if (Mask) {
2994 const auto *C = dyn_cast<Constant>(Mask);
2995 if (!C || !C->isAllOnesValue())
2996 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2997 }
2998
2999 if (NumElts < 8) {
3000 int Indices[8];
3001 for (unsigned i = 0; i != NumElts; ++i)
3002 Indices[i] = i;
3003 for (unsigned i = NumElts; i != 8; ++i)
3004 Indices[i] = NumElts + i % NumElts;
3005 Vec = Builder.CreateShuffleVector(Vec,
3006 Constant::getNullValue(Vec->getType()),
3007 Indices);
3008 }
3009 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
3010}
3011
3012static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
3013 unsigned CC, bool Signed) {
3014 Value *Op0 = CI.getArgOperand(0);
3015 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
3016
3017 Value *Cmp;
3018 if (CC == 3) {
3019 Cmp = Constant::getNullValue(
3020 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3021 } else if (CC == 7) {
3022 Cmp = Constant::getAllOnesValue(
3023 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3024 } else {
3025 ICmpInst::Predicate Pred;
3026 switch (CC) {
3027 default: llvm_unreachable("Unknown condition code");
3028 case 0: Pred = ICmpInst::ICMP_EQ; break;
3029 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
3030 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
3031 case 4: Pred = ICmpInst::ICMP_NE; break;
3032 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
3033 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
3034 }
3035 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
3036 }
3037
3038 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
3039
3040 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
3041}
3042
3043// Replace a masked intrinsic with an older unmasked intrinsic.
3044static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
3045 Intrinsic::ID IID) {
3046 Value *Rep =
3047 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
3048 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
3049}
3050
3051static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3052 Value* A = CI.getArgOperand(0);
3053 Value* B = CI.getArgOperand(1);
3054 Value* Src = CI.getArgOperand(2);
3055 Value* Mask = CI.getArgOperand(3);
3056
3057 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
3058 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3059 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
3060 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
3061 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3062 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3063}
3064
3065static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3066 Value* Op = CI.getArgOperand(0);
3067 Type* ReturnOp = CI.getType();
3068 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3069 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3070 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3071}
3072
3073// Replace intrinsic with unmasked version and a select.
3074static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3075 CallBase &CI, Value *&Rep) {
3076 Name = Name.substr(12); // Remove avx512.mask.
3077
3078 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3079 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3080 Intrinsic::ID IID;
3081 if (Name.starts_with("max.p")) {
3082 if (VecWidth == 128 && EltWidth == 32)
3083 IID = Intrinsic::x86_sse_max_ps;
3084 else if (VecWidth == 128 && EltWidth == 64)
3085 IID = Intrinsic::x86_sse2_max_pd;
3086 else if (VecWidth == 256 && EltWidth == 32)
3087 IID = Intrinsic::x86_avx_max_ps_256;
3088 else if (VecWidth == 256 && EltWidth == 64)
3089 IID = Intrinsic::x86_avx_max_pd_256;
3090 else
3091 llvm_unreachable("Unexpected intrinsic");
3092 } else if (Name.starts_with("min.p")) {
3093 if (VecWidth == 128 && EltWidth == 32)
3094 IID = Intrinsic::x86_sse_min_ps;
3095 else if (VecWidth == 128 && EltWidth == 64)
3096 IID = Intrinsic::x86_sse2_min_pd;
3097 else if (VecWidth == 256 && EltWidth == 32)
3098 IID = Intrinsic::x86_avx_min_ps_256;
3099 else if (VecWidth == 256 && EltWidth == 64)
3100 IID = Intrinsic::x86_avx_min_pd_256;
3101 else
3102 llvm_unreachable("Unexpected intrinsic");
3103 } else if (Name.starts_with("pshuf.b.")) {
3104 if (VecWidth == 128)
3105 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3106 else if (VecWidth == 256)
3107 IID = Intrinsic::x86_avx2_pshuf_b;
3108 else if (VecWidth == 512)
3109 IID = Intrinsic::x86_avx512_pshuf_b_512;
3110 else
3111 llvm_unreachable("Unexpected intrinsic");
3112 } else if (Name.starts_with("pmul.hr.sw.")) {
3113 if (VecWidth == 128)
3114 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3115 else if (VecWidth == 256)
3116 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3117 else if (VecWidth == 512)
3118 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3119 else
3120 llvm_unreachable("Unexpected intrinsic");
3121 } else if (Name.starts_with("pmulh.w")) {
3122 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3123 "Unexpected intrinsic");
3124 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::smulh);
3125 return true;
3126 } else if (Name.starts_with("pmulhu.w")) {
3127 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3128 "Unexpected intrinsic");
3129 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::umulh);
3130 return true;
3131 } else if (Name.starts_with("pmaddw.d.")) {
3132 if (VecWidth == 128)
3133 IID = Intrinsic::x86_sse2_pmadd_wd;
3134 else if (VecWidth == 256)
3135 IID = Intrinsic::x86_avx2_pmadd_wd;
3136 else if (VecWidth == 512)
3137 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3138 else
3139 llvm_unreachable("Unexpected intrinsic");
3140 } else if (Name.starts_with("pmaddubs.w.")) {
3141 if (VecWidth == 128)
3142 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3143 else if (VecWidth == 256)
3144 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3145 else if (VecWidth == 512)
3146 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3147 else
3148 llvm_unreachable("Unexpected intrinsic");
3149 } else if (Name.starts_with("packsswb.")) {
3150 if (VecWidth == 128)
3151 IID = Intrinsic::x86_sse2_packsswb_128;
3152 else if (VecWidth == 256)
3153 IID = Intrinsic::x86_avx2_packsswb;
3154 else if (VecWidth == 512)
3155 IID = Intrinsic::x86_avx512_packsswb_512;
3156 else
3157 llvm_unreachable("Unexpected intrinsic");
3158 } else if (Name.starts_with("packssdw.")) {
3159 if (VecWidth == 128)
3160 IID = Intrinsic::x86_sse2_packssdw_128;
3161 else if (VecWidth == 256)
3162 IID = Intrinsic::x86_avx2_packssdw;
3163 else if (VecWidth == 512)
3164 IID = Intrinsic::x86_avx512_packssdw_512;
3165 else
3166 llvm_unreachable("Unexpected intrinsic");
3167 } else if (Name.starts_with("packuswb.")) {
3168 if (VecWidth == 128)
3169 IID = Intrinsic::x86_sse2_packuswb_128;
3170 else if (VecWidth == 256)
3171 IID = Intrinsic::x86_avx2_packuswb;
3172 else if (VecWidth == 512)
3173 IID = Intrinsic::x86_avx512_packuswb_512;
3174 else
3175 llvm_unreachable("Unexpected intrinsic");
3176 } else if (Name.starts_with("packusdw.")) {
3177 if (VecWidth == 128)
3178 IID = Intrinsic::x86_sse41_packusdw;
3179 else if (VecWidth == 256)
3180 IID = Intrinsic::x86_avx2_packusdw;
3181 else if (VecWidth == 512)
3182 IID = Intrinsic::x86_avx512_packusdw_512;
3183 else
3184 llvm_unreachable("Unexpected intrinsic");
3185 } else if (Name.starts_with("vpermilvar.")) {
3186 if (VecWidth == 128 && EltWidth == 32)
3187 IID = Intrinsic::x86_avx_vpermilvar_ps;
3188 else if (VecWidth == 128 && EltWidth == 64)
3189 IID = Intrinsic::x86_avx_vpermilvar_pd;
3190 else if (VecWidth == 256 && EltWidth == 32)
3191 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3192 else if (VecWidth == 256 && EltWidth == 64)
3193 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3194 else if (VecWidth == 512 && EltWidth == 32)
3195 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3196 else if (VecWidth == 512 && EltWidth == 64)
3197 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3198 else
3199 llvm_unreachable("Unexpected intrinsic");
3200 } else if (Name == "cvtpd2dq.256") {
3201 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3202 } else if (Name == "cvtpd2ps.256") {
3203 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3204 } else if (Name == "cvttpd2dq.256") {
3205 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3206 } else if (Name == "cvttps2dq.128") {
3207 IID = Intrinsic::x86_sse2_cvttps2dq;
3208 } else if (Name == "cvttps2dq.256") {
3209 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3210 } else if (Name.starts_with("permvar.")) {
3211 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3212 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3213 IID = Intrinsic::x86_avx2_permps;
3214 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3215 IID = Intrinsic::x86_avx2_permd;
3216 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3217 IID = Intrinsic::x86_avx512_permvar_df_256;
3218 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3219 IID = Intrinsic::x86_avx512_permvar_di_256;
3220 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3221 IID = Intrinsic::x86_avx512_permvar_sf_512;
3222 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3223 IID = Intrinsic::x86_avx512_permvar_si_512;
3224 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3225 IID = Intrinsic::x86_avx512_permvar_df_512;
3226 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3227 IID = Intrinsic::x86_avx512_permvar_di_512;
3228 else if (VecWidth == 128 && EltWidth == 16)
3229 IID = Intrinsic::x86_avx512_permvar_hi_128;
3230 else if (VecWidth == 256 && EltWidth == 16)
3231 IID = Intrinsic::x86_avx512_permvar_hi_256;
3232 else if (VecWidth == 512 && EltWidth == 16)
3233 IID = Intrinsic::x86_avx512_permvar_hi_512;
3234 else if (VecWidth == 128 && EltWidth == 8)
3235 IID = Intrinsic::x86_avx512_permvar_qi_128;
3236 else if (VecWidth == 256 && EltWidth == 8)
3237 IID = Intrinsic::x86_avx512_permvar_qi_256;
3238 else if (VecWidth == 512 && EltWidth == 8)
3239 IID = Intrinsic::x86_avx512_permvar_qi_512;
3240 else
3241 llvm_unreachable("Unexpected intrinsic");
3242 } else if (Name.starts_with("dbpsadbw.")) {
3243 if (VecWidth == 128)
3244 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3245 else if (VecWidth == 256)
3246 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3247 else if (VecWidth == 512)
3248 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3249 else
3250 llvm_unreachable("Unexpected intrinsic");
3251 } else if (Name.starts_with("pmultishift.qb.")) {
3252 if (VecWidth == 128)
3253 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3254 else if (VecWidth == 256)
3255 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3256 else if (VecWidth == 512)
3257 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3258 else
3259 llvm_unreachable("Unexpected intrinsic");
3260 } else if (Name.starts_with("conflict.")) {
3261 if (Name[9] == 'd' && VecWidth == 128)
3262 IID = Intrinsic::x86_avx512_conflict_d_128;
3263 else if (Name[9] == 'd' && VecWidth == 256)
3264 IID = Intrinsic::x86_avx512_conflict_d_256;
3265 else if (Name[9] == 'd' && VecWidth == 512)
3266 IID = Intrinsic::x86_avx512_conflict_d_512;
3267 else if (Name[9] == 'q' && VecWidth == 128)
3268 IID = Intrinsic::x86_avx512_conflict_q_128;
3269 else if (Name[9] == 'q' && VecWidth == 256)
3270 IID = Intrinsic::x86_avx512_conflict_q_256;
3271 else if (Name[9] == 'q' && VecWidth == 512)
3272 IID = Intrinsic::x86_avx512_conflict_q_512;
3273 else
3274 llvm_unreachable("Unexpected intrinsic");
3275 } else if (Name.starts_with("pavg.")) {
3276 if (Name[5] == 'b' && VecWidth == 128)
3277 IID = Intrinsic::x86_sse2_pavg_b;
3278 else if (Name[5] == 'b' && VecWidth == 256)
3279 IID = Intrinsic::x86_avx2_pavg_b;
3280 else if (Name[5] == 'b' && VecWidth == 512)
3281 IID = Intrinsic::x86_avx512_pavg_b_512;
3282 else if (Name[5] == 'w' && VecWidth == 128)
3283 IID = Intrinsic::x86_sse2_pavg_w;
3284 else if (Name[5] == 'w' && VecWidth == 256)
3285 IID = Intrinsic::x86_avx2_pavg_w;
3286 else if (Name[5] == 'w' && VecWidth == 512)
3287 IID = Intrinsic::x86_avx512_pavg_w_512;
3288 else
3289 llvm_unreachable("Unexpected intrinsic");
3290 } else
3291 return false;
3292
3293 SmallVector<Value *, 4> Args(CI.args());
3294 Args.pop_back();
3295 Args.pop_back();
3296 Rep = Builder.CreateIntrinsic(IID, Args);
3297 unsigned NumArgs = CI.arg_size();
3298 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3299 CI.getArgOperand(NumArgs - 2));
3300 return true;
3301}
3302
3303/// Upgrade comment in call to inline asm that represents an objc retain release
3304/// marker.
3305void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3306 size_t Pos;
3307 if (AsmStr->find("mov\tfp") == 0 &&
3308 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3309 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3310 AsmStr->replace(Pos, 1, ";");
3311 }
3312}
3313
3314static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3315 StringRef Name,
3316 const Intrinsic::ID (&IIDs)[2][3]) {
3317 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3318 assert(Result && "unsupported nvvm.{add,mul,fma}.* intrinsic");
3319 auto [IID, RoundingMode] = *Result;
3320
3321 // The legacy bf16 variants used integer storage types for the bf16
3322 // operands; bitcast them to the floating-point types of the new
3323 // overloaded intrinsic.
3324 Type *Ty = CI->getType();
3325 const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
3326 if (IsLegacyBF16Storage)
3327 Ty = Name.ends_with("bf16x2")
3328 ? FixedVectorType::get(Builder.getBFloatTy(), 2)
3329 : static_cast<Type *>(Builder.getBFloatTy());
3330
3331 SmallVector<Value *, 4> Args;
3332 for (Value *Arg : CI->args())
3333 Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty) : Arg);
3334 Args.push_back(Builder.getInt32(static_cast<int>(RoundingMode)));
3335
3336 Value *Rep = Builder.CreateIntrinsic(Ty, IID, Args);
3337 return IsLegacyBF16Storage ? Builder.CreateBitCast(Rep, CI->getType()) : Rep;
3338}
3339
3340static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3341 Function *F, IRBuilder<> &Builder) {
3342 Value *Rep = nullptr;
3343
3344 if (Name == "abs.i" || Name == "abs.ll") {
3345 Value *Arg = CI->getArgOperand(0);
3346 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3347 {Arg, Builder.getTrue()},
3348 /*FMFSource=*/nullptr, "abs");
3349 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3350 Type *Ty = (Name == "abs.bf16")
3351 ? Builder.getBFloatTy()
3352 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3353 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3354 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3355 Rep = Builder.CreateBitCast(Abs, CI->getType());
3356 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3357 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3358 : Intrinsic::nvvm_fabs;
3359 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3360 } else if (Name.consume_front("add.")) {
3361 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3362 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFAddIIDs);
3363 } else if (Name.consume_front("mul.")) {
3364 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3365 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMulIIDs);
3366 } else if (Name.consume_front("ex2.approx.")) {
3367 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3368 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3369 : Intrinsic::nvvm_ex2_approx;
3370 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3371 } else if (Name.consume_front("fma.")) {
3372 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
3373 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMAIIDs);
3374 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3375 Name.starts_with("atomic.load.add.f64.p")) {
3376 Value *Ptr = CI->getArgOperand(0);
3377 Value *Val = CI->getArgOperand(1);
3378 Rep = Builder.CreateAtomicRMW(
3379 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3380 CI->getContext().getOrInsertSyncScopeID("device"));
3381 cast<Instruction>(Rep)->setMetadata(
3382 LLVMContext::MD_atomic_ignore_denormal_mode,
3383 MDNode::get(CI->getContext(), {}));
3384 // The default scope for atomic.load.* intrinsics is device
3385 // (= gpu scope in ptx), but the default LLVM atomic scope is
3386 // "system"
3387 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3388 Name.starts_with("atomic.load.dec.32.p")) {
3389 Value *Ptr = CI->getArgOperand(0);
3390 Value *Val = CI->getArgOperand(1);
3391 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3392 : AtomicRMWInst::UDecWrap;
3393 Rep = Builder.CreateAtomicRMW(
3394 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3395 CI->getContext().getOrInsertSyncScopeID("device"));
3396 // See comment above.
3397 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3398 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3399 StringRef Op = Name.substr(StringRef("atomic.").size());
3400 Value *Ptr = CI->getArgOperand(0);
3401 Value *Val = CI->getArgOperand(1);
3402 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3403 Op.contains(".cta.") ? "block" : "");
3404 if (Op.starts_with("cas.")) {
3405 Value *New = CI->getArgOperand(2);
3406 Value *Pair = Builder.CreateAtomicCmpXchg(
3407 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3408 AtomicOrdering::Monotonic, SSID);
3409 Rep = Builder.CreateExtractValue(Pair, 0);
3410 } else {
3411 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3412 // because we were actually missing those intrinsics!
3413 AtomicRMWInst::BinOp BinOp =
3414 StringSwitch<AtomicRMWInst::BinOp>(Op)
3415 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3416 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3417 .StartsWith("exch.", AtomicRMWInst::Xchg)
3418 .StartsWith("max.", AtomicRMWInst::Max)
3419 .StartsWith("min.", AtomicRMWInst::Min)
3420 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3421 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3422 .StartsWith("and.", AtomicRMWInst::And)
3423 .StartsWith("or.", AtomicRMWInst::Or)
3424 .StartsWith("xor.", AtomicRMWInst::Xor)
3425 .Default(AtomicRMWInst::BAD_BINOP);
3426 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3427 "unexpected nvvm scoped atomic intrinsic");
3428 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3429 AtomicOrdering::Monotonic, SSID);
3430 if (BinOp == AtomicRMWInst::FAdd) {
3431 cast<Instruction>(Rep)->setMetadata(
3432 LLVMContext::MD_atomic_ignore_denormal_mode,
3433 MDNode::get(CI->getContext(), {}));
3434 }
3435 }
3436 } else if (Name == "clz.ll") {
3437 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3438 Value *Arg = CI->getArgOperand(0);
3439 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3440 {Arg, Builder.getFalse()},
3441 /*FMFSource=*/nullptr, "ctlz");
3442 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3443 } else if (Name == "popc.ll") {
3444 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3445 // i64.
3446 Value *Arg = CI->getArgOperand(0);
3447 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3448 Arg, /*FMFSource=*/nullptr, "ctpop");
3449 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3450 } else if (Name == "h2f") {
3451 Value *Cast =
3452 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3453 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3454 } else if (Name.consume_front("bitcast.") &&
3455 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3456 Name == "d2ll")) {
3457 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3458 } else if (Name == "rotate.b32") {
3459 Value *Arg = CI->getOperand(0);
3460 Value *ShiftAmt = CI->getOperand(1);
3461 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3462 {Arg, Arg, ShiftAmt});
3463 } else if (Name == "rotate.b64") {
3464 Type *Int64Ty = Builder.getInt64Ty();
3465 Value *Arg = CI->getOperand(0);
3466 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3467 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3468 {Arg, Arg, ZExtShiftAmt});
3469 } else if (Name == "rotate.right.b64") {
3470 Type *Int64Ty = Builder.getInt64Ty();
3471 Value *Arg = CI->getOperand(0);
3472 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3473 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3474 {Arg, Arg, ZExtShiftAmt});
3475 } else if (Name == "swap.lo.hi.b64") {
3476 Type *Int64Ty = Builder.getInt64Ty();
3477 Value *Arg = CI->getOperand(0);
3478 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3479 {Arg, Arg, Builder.getInt64(32)});
3480 } else if ((Name.consume_front("ptr.gen.to.") &&
3481 consumeNVVMPtrAddrSpace(Name)) ||
3482 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3483 Name.starts_with(".to.gen"))) {
3484 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3485 } else if (Name.consume_front("ldg.global")) {
3486 Value *Ptr = CI->getArgOperand(0);
3487 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3488 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3489 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3490 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3491 MDNode *MD = MDNode::get(Builder.getContext(), {});
3492 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3493 return LD;
3494 } else if (Name == "tanh.approx.f32") {
3495 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3496 FastMathFlags FMF;
3497 FMF.setApproxFunc();
3498 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3499 FMF);
3500 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3501 Value *Arg =
3502 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3503 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3504 {}, {Arg});
3505 } else if (Name == "barrier") {
3506 Rep = Builder.CreateIntrinsic(
3507 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3508 {CI->getArgOperand(0), CI->getArgOperand(1)});
3509 } else if (Name == "barrier.sync") {
3510 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3511 {CI->getArgOperand(0)});
3512 } else if (Name == "barrier.sync.cnt") {
3513 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3514 {CI->getArgOperand(0), CI->getArgOperand(1)});
3515 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3516 Name == "barrier0.or") {
3517 Value *C = CI->getArgOperand(0);
3518 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3519
3520 Intrinsic::ID IID =
3521 StringSwitch<Intrinsic::ID>(Name)
3522 .Case("barrier0.popc",
3523 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3524 .Case("barrier0.and",
3525 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3526 .Case("barrier0.or",
3527 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3528 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3529 Rep = Builder.CreateZExt(Bar, CI->getType());
3530 } else {
3531 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3532 if (IID != Intrinsic::not_intrinsic &&
3533 isLegacyNVPTXBF16IntSignature(F, IID)) {
3534 rename(F);
3535 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3536 SmallVector<Value *, 2> Args;
3537 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3538 Value *Arg = CI->getArgOperand(I);
3539 Type *OldType = Arg->getType();
3540 Type *NewType = NewFn->getArg(I)->getType();
3541 Args.push_back(
3542 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3543 ? Builder.CreateBitCast(Arg, NewType)
3544 : Arg);
3545 }
3546 Rep = Builder.CreateCall(NewFn, Args);
3547 if (F->getReturnType()->isIntegerTy())
3548 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3549 }
3550 }
3551
3552 return Rep;
3553}
3554
3555static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3556 IRBuilder<> &Builder) {
3557 LLVMContext &C = F->getContext();
3558 Value *Rep = nullptr;
3559
3560 if (Name.starts_with("sse4a.movnt.")) {
3561 SmallVector<Metadata *, 1> Elts;
3562 Elts.push_back(
3563 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3564 MDNode *Node = MDNode::get(C, Elts);
3565
3566 Value *Arg0 = CI->getArgOperand(0);
3567 Value *Arg1 = CI->getArgOperand(1);
3568
3569 // Nontemporal (unaligned) store of the 0'th element of the float/double
3570 // vector.
3571 Value *Extract =
3572 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3573
3574 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3575 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3576 } else if (Name.starts_with("avx.movnt.") ||
3577 Name.starts_with("avx512.storent.")) {
3578 SmallVector<Metadata *, 1> Elts;
3579 Elts.push_back(
3580 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3581 MDNode *Node = MDNode::get(C, Elts);
3582
3583 Value *Arg0 = CI->getArgOperand(0);
3584 Value *Arg1 = CI->getArgOperand(1);
3585
3586 StoreInst *SI = Builder.CreateAlignedStore(
3587 Arg1, Arg0,
3588 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3589 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3590 } else if (Name == "sse2.storel.dq") {
3591 Value *Arg0 = CI->getArgOperand(0);
3592 Value *Arg1 = CI->getArgOperand(1);
3593
3594 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3595 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3596 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3597 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3598 } else if (Name.starts_with("sse.storeu.") ||
3599 Name.starts_with("sse2.storeu.") ||
3600 Name.starts_with("avx.storeu.")) {
3601 Value *Arg0 = CI->getArgOperand(0);
3602 Value *Arg1 = CI->getArgOperand(1);
3603 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3604 } else if (Name == "avx512.mask.store.ss") {
3605 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3606 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3607 Mask, false);
3608 } else if (Name.starts_with("avx512.mask.store")) {
3609 // "avx512.mask.storeu." or "avx512.mask.store."
3610 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3611 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3612 CI->getArgOperand(2), Aligned);
3613 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3614 // Upgrade packed integer vector compare intrinsics to compare instructions.
3615 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3616 bool CmpEq = Name[9] == 'e';
3617 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3618 CI->getArgOperand(0), CI->getArgOperand(1));
3619 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3620 } else if (Name.starts_with("avx512.broadcastm")) {
3621 Type *ExtTy = Type::getInt32Ty(C);
3622 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3623 ExtTy = Type::getInt64Ty(C);
3624 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3625 ExtTy->getPrimitiveSizeInBits();
3626 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3627 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3628 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3629 Value *Vec = CI->getArgOperand(0);
3630 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3631 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3632 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3633 } else if (Name.starts_with("avx.sqrt.p") ||
3634 Name.starts_with("sse2.sqrt.p") ||
3635 Name.starts_with("sse.sqrt.p")) {
3636 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3637 {CI->getArgOperand(0)});
3638 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3639 if (CI->arg_size() == 4 &&
3640 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3641 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3642 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3643 : Intrinsic::x86_avx512_sqrt_pd_512;
3644
3645 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3646 Rep = Builder.CreateIntrinsic(IID, Args);
3647 } else {
3648 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3649 {CI->getArgOperand(0)});
3650 }
3651 Rep =
3652 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3653 } else if (Name.starts_with("avx512.ptestm") ||
3654 Name.starts_with("avx512.ptestnm")) {
3655 Value *Op0 = CI->getArgOperand(0);
3656 Value *Op1 = CI->getArgOperand(1);
3657 Value *Mask = CI->getArgOperand(2);
3658 Rep = Builder.CreateAnd(Op0, Op1);
3659 llvm::Type *Ty = Op0->getType();
3660 Value *Zero = llvm::Constant::getNullValue(Ty);
3661 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3662 ? ICmpInst::ICMP_NE
3663 : ICmpInst::ICMP_EQ;
3664 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3665 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3666 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3667 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3668 ->getNumElements();
3669 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3670 Rep =
3671 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3672 } else if (Name.starts_with("avx512.kunpck")) {
3673 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3674 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3675 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3676 int Indices[64];
3677 for (unsigned i = 0; i != NumElts; ++i)
3678 Indices[i] = i;
3679
3680 // First extract half of each vector. This gives better codegen than
3681 // doing it in a single shuffle.
3682 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3683 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3684 // Concat the vectors.
3685 // NOTE: Operands have to be swapped to match intrinsic definition.
3686 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3687 Rep = Builder.CreateBitCast(Rep, CI->getType());
3688 } else if (Name == "avx512.kand.w") {
3689 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3690 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3691 Rep = Builder.CreateAnd(LHS, RHS);
3692 Rep = Builder.CreateBitCast(Rep, CI->getType());
3693 } else if (Name == "avx512.kandn.w") {
3694 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3695 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3696 LHS = Builder.CreateNot(LHS);
3697 Rep = Builder.CreateAnd(LHS, RHS);
3698 Rep = Builder.CreateBitCast(Rep, CI->getType());
3699 } else if (Name == "avx512.kor.w") {
3700 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3701 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3702 Rep = Builder.CreateOr(LHS, RHS);
3703 Rep = Builder.CreateBitCast(Rep, CI->getType());
3704 } else if (Name == "avx512.kxor.w") {
3705 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3706 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3707 Rep = Builder.CreateXor(LHS, RHS);
3708 Rep = Builder.CreateBitCast(Rep, CI->getType());
3709 } else if (Name == "avx512.kxnor.w") {
3710 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3711 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3712 LHS = Builder.CreateNot(LHS);
3713 Rep = Builder.CreateXor(LHS, RHS);
3714 Rep = Builder.CreateBitCast(Rep, CI->getType());
3715 } else if (Name == "avx512.knot.w") {
3716 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3717 Rep = Builder.CreateNot(Rep);
3718 Rep = Builder.CreateBitCast(Rep, CI->getType());
3719 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3720 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3721 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3722 Rep = Builder.CreateOr(LHS, RHS);
3723 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3724 Value *C;
3725 if (Name[14] == 'c')
3726 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3727 else
3728 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3729 Rep = Builder.CreateICmpEQ(Rep, C);
3730 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3731 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3732 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3733 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3734 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3735 Type *I32Ty = Type::getInt32Ty(C);
3736 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3737 ConstantInt::get(I32Ty, 0));
3738 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3739 ConstantInt::get(I32Ty, 0));
3740 Value *EltOp;
3741 if (Name.contains(".add."))
3742 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3743 else if (Name.contains(".sub."))
3744 EltOp = Builder.CreateFSub(Elt0, Elt1);
3745 else if (Name.contains(".mul."))
3746 EltOp = Builder.CreateFMul(Elt0, Elt1);
3747 else
3748 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3749 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3750 ConstantInt::get(I32Ty, 0));
3751 } else if (Name.starts_with("avx512.mask.pcmp")) {
3752 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3753 bool CmpEq = Name[16] == 'e';
3754 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3755 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3756 Type *OpTy = CI->getArgOperand(0)->getType();
3757 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3758 Intrinsic::ID IID;
3759 switch (VecWidth) {
3760 default:
3761 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3762 break;
3763 case 128:
3764 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3765 break;
3766 case 256:
3767 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3768 break;
3769 case 512:
3770 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3771 break;
3772 }
3773
3774 Rep =
3775 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3776 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3777 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3778 Type *OpTy = CI->getArgOperand(0)->getType();
3779 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3780 unsigned EltWidth = OpTy->getScalarSizeInBits();
3781 Intrinsic::ID IID;
3782 if (VecWidth == 128 && EltWidth == 32)
3783 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3784 else if (VecWidth == 256 && EltWidth == 32)
3785 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3786 else if (VecWidth == 512 && EltWidth == 32)
3787 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3788 else if (VecWidth == 128 && EltWidth == 64)
3789 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3790 else if (VecWidth == 256 && EltWidth == 64)
3791 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3792 else if (VecWidth == 512 && EltWidth == 64)
3793 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3794 else
3795 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3796
3797 Rep =
3798 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3799 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3800 } else if (Name.starts_with("avx512.cmp.p")) {
3801 SmallVector<Value *, 4> Args(CI->args());
3802 Type *OpTy = Args[0]->getType();
3803 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3804 unsigned EltWidth = OpTy->getScalarSizeInBits();
3805 Intrinsic::ID IID;
3806 if (VecWidth == 128 && EltWidth == 32)
3807 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3808 else if (VecWidth == 256 && EltWidth == 32)
3809 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3810 else if (VecWidth == 512 && EltWidth == 32)
3811 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3812 else if (VecWidth == 128 && EltWidth == 64)
3813 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3814 else if (VecWidth == 256 && EltWidth == 64)
3815 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3816 else if (VecWidth == 512 && EltWidth == 64)
3817 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3818 else
3819 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3820
3821 Value *Mask = Constant::getAllOnesValue(CI->getType());
3822 if (VecWidth == 512)
3823 std::swap(Mask, Args.back());
3824 Args.push_back(Mask);
3825
3826 Rep = Builder.CreateIntrinsic(IID, Args);
3827 } else if (Name.starts_with("avx512.mask.cmp.")) {
3828 // Integer compare intrinsics.
3829 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3830 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3831 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3832 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3833 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3834 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3835 Name.starts_with("avx512.cvtw2mask.") ||
3836 Name.starts_with("avx512.cvtd2mask.") ||
3837 Name.starts_with("avx512.cvtq2mask.")) {
3838 Value *Op = CI->getArgOperand(0);
3839 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3840 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3841 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3842 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3843 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3844 Name.starts_with("avx512.mask.pabs")) {
3845 Rep = upgradeAbs(Builder, *CI);
3846 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3847 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3848 Name.starts_with("avx512.mask.pmaxs")) {
3849 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3850 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3851 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3852 Name.starts_with("avx512.mask.pmaxu")) {
3853 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3854 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3855 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3856 Name.starts_with("avx512.mask.pmins")) {
3857 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3858 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3859 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3860 Name.starts_with("avx512.mask.pminu")) {
3861 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3862 } else if (Name == "sse2.pmulh.w" || Name.starts_with("avx2.pmulh.w") ||
3863 Name.starts_with("avx512.pmulh.w")) {
3864 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smulh);
3865 } else if (Name == "sse2.pmulhu.w" || Name.starts_with("avx2.pmulhu.w") ||
3866 Name.starts_with("avx512.pmulhu.w")) {
3867 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umulh);
3868 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3869 Name == "avx512.pmulu.dq.512" ||
3870 Name.starts_with("avx512.mask.pmulu.dq.")) {
3871 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3872 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3873 Name == "avx512.pmul.dq.512" ||
3874 Name.starts_with("avx512.mask.pmul.dq.")) {
3875 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3876 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3877 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3878 Rep =
3879 Builder.CreateSIToFP(CI->getArgOperand(1),
3880 cast<VectorType>(CI->getType())->getElementType());
3881 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3882 } else if (Name == "avx512.cvtusi2sd") {
3883 Rep =
3884 Builder.CreateUIToFP(CI->getArgOperand(1),
3885 cast<VectorType>(CI->getType())->getElementType());
3886 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3887 } else if (Name == "sse2.cvtss2sd") {
3888 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3889 Rep = Builder.CreateFPExt(
3890 Rep, cast<VectorType>(CI->getType())->getElementType());
3891 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3892 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3893 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3894 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3895 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3896 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3897 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3898 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3899 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3900 Name == "avx512.mask.cvtqq2ps.256" ||
3901 Name == "avx512.mask.cvtqq2ps.512" ||
3902 Name == "avx512.mask.cvtuqq2ps.256" ||
3903 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3904 Name == "avx.cvt.ps2.pd.256" ||
3905 Name == "avx512.mask.cvtps2pd.128" ||
3906 Name == "avx512.mask.cvtps2pd.256") {
3907 auto *DstTy = cast<FixedVectorType>(CI->getType());
3908 Rep = CI->getArgOperand(0);
3909 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3910
3911 unsigned NumDstElts = DstTy->getNumElements();
3912 if (NumDstElts < SrcTy->getNumElements()) {
3913 assert(NumDstElts == 2 && "Unexpected vector size");
3914 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3915 }
3916
3917 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3918 bool IsUnsigned = Name.contains("cvtu");
3919 if (IsPS2PD)
3920 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3921 else if (CI->arg_size() == 4 &&
3922 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3923 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3924 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3925 : Intrinsic::x86_avx512_sitofp_round;
3926 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3927 {Rep, CI->getArgOperand(3)});
3928 } else {
3929 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3930 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3931 }
3932
3933 if (CI->arg_size() >= 3)
3934 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3935 CI->getArgOperand(1));
3936 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3937 Name.starts_with("vcvtph2ps.")) {
3938 auto *DstTy = cast<FixedVectorType>(CI->getType());
3939 Rep = CI->getArgOperand(0);
3940 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3941 unsigned NumDstElts = DstTy->getNumElements();
3942 if (NumDstElts != SrcTy->getNumElements()) {
3943 assert(NumDstElts == 4 && "Unexpected vector size");
3944 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3945 }
3946 Rep = Builder.CreateBitCast(
3947 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3948 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3949 if (CI->arg_size() >= 3)
3950 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3951 CI->getArgOperand(1));
3952 } else if (Name.starts_with("avx512.mask.load")) {
3953 // "avx512.mask.loadu." or "avx512.mask.load."
3954 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3955 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3956 CI->getArgOperand(2), Aligned);
3957 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3958 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3959 auto *PtrTy = CI->getOperand(0)->getType();
3960 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3961 ResultTy->getNumElements());
3962 Rep = Builder.CreateIntrinsic(
3963 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3964 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3965 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3966 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3967 auto *PtrTy = CI->getArgOperand(0)->getType();
3968 Value *MaskVec =
3969 getX86MaskVec(Builder, CI->getArgOperand(2),
3970 cast<FixedVectorType>(ResultTy)->getNumElements());
3971 Rep = Builder.CreateIntrinsic(
3972 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3973 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3974 } else if (Name.starts_with("avx512.mask.compress.") ||
3975 Name.starts_with("avx512.mask.expand.")) {
3976 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3977
3978 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3979 ResultTy->getNumElements());
3980
3981 bool IsCompress = Name[12] == 'c';
3982 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3983 : Intrinsic::x86_avx512_mask_expand;
3984 Rep = Builder.CreateIntrinsic(
3985 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3986 } else if (Name.starts_with("xop.vpcom")) {
3987 bool IsSigned;
3988 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3989 Name.ends_with("uq"))
3990 IsSigned = false;
3991 else if (Name.ends_with("b") || Name.ends_with("w") ||
3992 Name.ends_with("d") || Name.ends_with("q"))
3993 IsSigned = true;
3994 else
3995 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3996
3997 unsigned Imm;
3998 if (CI->arg_size() == 3) {
3999 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4000 } else {
4001 Name = Name.substr(9); // strip off "xop.vpcom"
4002 if (Name.starts_with("lt"))
4003 Imm = 0;
4004 else if (Name.starts_with("le"))
4005 Imm = 1;
4006 else if (Name.starts_with("gt"))
4007 Imm = 2;
4008 else if (Name.starts_with("ge"))
4009 Imm = 3;
4010 else if (Name.starts_with("eq"))
4011 Imm = 4;
4012 else if (Name.starts_with("ne"))
4013 Imm = 5;
4014 else if (Name.starts_with("false"))
4015 Imm = 6;
4016 else if (Name.starts_with("true"))
4017 Imm = 7;
4018 else
4019 llvm_unreachable("Unknown condition");
4020 }
4021
4022 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
4023 } else if (Name.starts_with("xop.vpcmov")) {
4024 Value *Sel = CI->getArgOperand(2);
4025 Value *NotSel = Builder.CreateNot(Sel);
4026 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
4027 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
4028 Rep = Builder.CreateOr(Sel0, Sel1);
4029 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
4030 Name.starts_with("avx512.mask.prol")) {
4031 Rep = upgradeX86Rotate(Builder, *CI, false);
4032 } else if (Name.starts_with("avx512.pror") ||
4033 Name.starts_with("avx512.mask.pror")) {
4034 Rep = upgradeX86Rotate(Builder, *CI, true);
4035 } else if (Name.starts_with("avx512.vpshld.") ||
4036 Name.starts_with("avx512.mask.vpshld") ||
4037 Name.starts_with("avx512.maskz.vpshld")) {
4038 bool ZeroMask = Name[11] == 'z';
4039 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
4040 } else if (Name.starts_with("avx512.vpshrd.") ||
4041 Name.starts_with("avx512.mask.vpshrd") ||
4042 Name.starts_with("avx512.maskz.vpshrd")) {
4043 bool ZeroMask = Name[11] == 'z';
4044 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
4045 } else if (Name == "sse42.crc32.64.8") {
4046 Value *Trunc0 =
4047 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
4048 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
4049 {Trunc0, CI->getArgOperand(1)});
4050 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
4051 } else if (Name.starts_with("avx.vbroadcast.s") ||
4052 Name.starts_with("avx512.vbroadcast.s")) {
4053 // Replace broadcasts with a series of insertelements.
4054 auto *VecTy = cast<FixedVectorType>(CI->getType());
4055 Type *EltTy = VecTy->getElementType();
4056 unsigned EltNum = VecTy->getNumElements();
4057 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
4058 Type *I32Ty = Type::getInt32Ty(C);
4059 Rep = PoisonValue::get(VecTy);
4060 for (unsigned I = 0; I < EltNum; ++I)
4061 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
4062 } else if (Name.starts_with("sse41.pmovsx") ||
4063 Name.starts_with("sse41.pmovzx") ||
4064 Name.starts_with("avx2.pmovsx") ||
4065 Name.starts_with("avx2.pmovzx") ||
4066 Name.starts_with("avx512.mask.pmovsx") ||
4067 Name.starts_with("avx512.mask.pmovzx")) {
4068 auto *DstTy = cast<FixedVectorType>(CI->getType());
4069 unsigned NumDstElts = DstTy->getNumElements();
4070
4071 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4072 SmallVector<int, 8> ShuffleMask(NumDstElts);
4073 for (unsigned i = 0; i != NumDstElts; ++i)
4074 ShuffleMask[i] = i;
4075
4076 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
4077
4078 bool DoSext = Name.contains("pmovsx");
4079 Rep =
4080 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4081 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4082 if (CI->arg_size() == 3)
4083 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4084 CI->getArgOperand(1));
4085 } else if (Name == "avx512.mask.pmov.qd.256" ||
4086 Name == "avx512.mask.pmov.qd.512" ||
4087 Name == "avx512.mask.pmov.wb.256" ||
4088 Name == "avx512.mask.pmov.wb.512") {
4089 Type *Ty = CI->getArgOperand(1)->getType();
4090 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
4091 Rep =
4092 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4093 } else if (Name.starts_with("avx.vbroadcastf128") ||
4094 Name == "avx2.vbroadcasti128") {
4095 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4096 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4097 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4098 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4099 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4100 if (NumSrcElts == 2)
4101 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4102 else
4103 Rep = Builder.CreateShuffleVector(Load,
4104 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4105 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4106 Name.starts_with("avx512.mask.shuf.f")) {
4107 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4108 Type *VT = CI->getType();
4109 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4110 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4111 unsigned ControlBitsMask = NumLanes - 1;
4112 unsigned NumControlBits = NumLanes / 2;
4113 SmallVector<int, 8> ShuffleMask(0);
4114
4115 for (unsigned l = 0; l != NumLanes; ++l) {
4116 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4117 // We actually need the other source.
4118 if (l >= NumLanes / 2)
4119 LaneMask += NumLanes;
4120 for (unsigned i = 0; i != NumElementsInLane; ++i)
4121 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4122 }
4123 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4124 CI->getArgOperand(1), ShuffleMask);
4125 Rep =
4126 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4127 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4128 Name.starts_with("avx512.mask.broadcasti")) {
4129 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4130 ->getNumElements();
4131 unsigned NumDstElts =
4132 cast<FixedVectorType>(CI->getType())->getNumElements();
4133
4134 SmallVector<int, 8> ShuffleMask(NumDstElts);
4135 for (unsigned i = 0; i != NumDstElts; ++i)
4136 ShuffleMask[i] = i % NumSrcElts;
4137
4138 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4139 CI->getArgOperand(0), ShuffleMask);
4140 Rep =
4141 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4142 } else if (Name.starts_with("avx2.pbroadcast") ||
4143 Name.starts_with("avx2.vbroadcast") ||
4144 Name.starts_with("avx512.pbroadcast") ||
4145 Name.starts_with("avx512.mask.broadcast.s")) {
4146 // Replace vp?broadcasts with a vector shuffle.
4147 Value *Op = CI->getArgOperand(0);
4148 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4149 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4150 SmallVector<int, 8> M;
4151 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4152 Rep = Builder.CreateShuffleVector(Op, M);
4153
4154 if (CI->arg_size() == 3)
4155 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4156 CI->getArgOperand(1));
4157 } else if (Name.starts_with("sse2.padds.") ||
4158 Name.starts_with("avx2.padds.") ||
4159 Name.starts_with("avx512.padds.") ||
4160 Name.starts_with("avx512.mask.padds.")) {
4161 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4162 } else if (Name.starts_with("sse2.psubs.") ||
4163 Name.starts_with("avx2.psubs.") ||
4164 Name.starts_with("avx512.psubs.") ||
4165 Name.starts_with("avx512.mask.psubs.")) {
4166 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4167 } else if (Name.starts_with("sse2.paddus.") ||
4168 Name.starts_with("avx2.paddus.") ||
4169 Name.starts_with("avx512.mask.paddus.")) {
4170 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4171 } else if (Name.starts_with("sse2.psubus.") ||
4172 Name.starts_with("avx2.psubus.") ||
4173 Name.starts_with("avx512.mask.psubus.")) {
4174 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4175 } else if (Name.starts_with("avx512.mask.palignr.")) {
4176 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4177 CI->getArgOperand(1), CI->getArgOperand(2),
4178 CI->getArgOperand(3), CI->getArgOperand(4),
4179 false);
4180 } else if (Name.starts_with("avx512.mask.valign.")) {
4181 Rep = upgradeX86ALIGNIntrinsics(
4182 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4183 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4184 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4185 // 128/256-bit shift left specified in bits.
4186 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4187 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4188 Shift / 8); // Shift is in bits.
4189 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4190 // 128/256-bit shift right specified in bits.
4191 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4192 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4193 Shift / 8); // Shift is in bits.
4194 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4195 Name == "avx512.psll.dq.512") {
4196 // 128/256/512-bit shift left specified in bytes.
4197 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4198 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4199 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4200 Name == "avx512.psrl.dq.512") {
4201 // 128/256/512-bit shift right specified in bytes.
4202 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4203 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4204 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4205 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4206 Name.starts_with("avx2.pblendd.")) {
4207 Value *Op0 = CI->getArgOperand(0);
4208 Value *Op1 = CI->getArgOperand(1);
4209 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4210 auto *VecTy = cast<FixedVectorType>(CI->getType());
4211 unsigned NumElts = VecTy->getNumElements();
4212
4213 SmallVector<int, 16> Idxs(NumElts);
4214 for (unsigned i = 0; i != NumElts; ++i)
4215 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4216
4217 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4218 } else if (Name.starts_with("avx.vinsertf128.") ||
4219 Name == "avx2.vinserti128" ||
4220 Name.starts_with("avx512.mask.insert")) {
4221 Value *Op0 = CI->getArgOperand(0);
4222 Value *Op1 = CI->getArgOperand(1);
4223 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4224 unsigned DstNumElts =
4225 cast<FixedVectorType>(CI->getType())->getNumElements();
4226 unsigned SrcNumElts =
4227 cast<FixedVectorType>(Op1->getType())->getNumElements();
4228 unsigned Scale = DstNumElts / SrcNumElts;
4229
4230 // Mask off the high bits of the immediate value; hardware ignores those.
4231 Imm = Imm % Scale;
4232
4233 // Extend the second operand into a vector the size of the destination.
4234 SmallVector<int, 8> Idxs(DstNumElts);
4235 for (unsigned i = 0; i != SrcNumElts; ++i)
4236 Idxs[i] = i;
4237 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4238 Idxs[i] = SrcNumElts;
4239 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4240
4241 // Insert the second operand into the first operand.
4242
4243 // Note that there is no guarantee that instruction lowering will actually
4244 // produce a vinsertf128 instruction for the created shuffles. In
4245 // particular, the 0 immediate case involves no lane changes, so it can
4246 // be handled as a blend.
4247
4248 // Example of shuffle mask for 32-bit elements:
4249 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4250 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4251
4252 // First fill with identify mask.
4253 for (unsigned i = 0; i != DstNumElts; ++i)
4254 Idxs[i] = i;
4255 // Then replace the elements where we need to insert.
4256 for (unsigned i = 0; i != SrcNumElts; ++i)
4257 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4258 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4259
4260 // If the intrinsic has a mask operand, handle that.
4261 if (CI->arg_size() == 5)
4262 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4263 CI->getArgOperand(3));
4264 } else if (Name.starts_with("avx.vextractf128.") ||
4265 Name == "avx2.vextracti128" ||
4266 Name.starts_with("avx512.mask.vextract")) {
4267 Value *Op0 = CI->getArgOperand(0);
4268 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4269 unsigned DstNumElts =
4270 cast<FixedVectorType>(CI->getType())->getNumElements();
4271 unsigned SrcNumElts =
4272 cast<FixedVectorType>(Op0->getType())->getNumElements();
4273 unsigned Scale = SrcNumElts / DstNumElts;
4274
4275 // Mask off the high bits of the immediate value; hardware ignores those.
4276 Imm = Imm % Scale;
4277
4278 // Get indexes for the subvector of the input vector.
4279 SmallVector<int, 8> Idxs(DstNumElts);
4280 for (unsigned i = 0; i != DstNumElts; ++i) {
4281 Idxs[i] = i + (Imm * DstNumElts);
4282 }
4283 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4284
4285 // If the intrinsic has a mask operand, handle that.
4286 if (CI->arg_size() == 4)
4287 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4288 CI->getArgOperand(2));
4289 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4290 Name.starts_with("avx512.mask.perm.di.")) {
4291 Value *Op0 = CI->getArgOperand(0);
4292 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4293 auto *VecTy = cast<FixedVectorType>(CI->getType());
4294 unsigned NumElts = VecTy->getNumElements();
4295
4296 SmallVector<int, 8> Idxs(NumElts);
4297 for (unsigned i = 0; i != NumElts; ++i)
4298 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4299
4300 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4301
4302 if (CI->arg_size() == 4)
4303 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4304 CI->getArgOperand(2));
4305 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4306 // The immediate permute control byte looks like this:
4307 // [1:0] - select 128 bits from sources for low half of destination
4308 // [2] - ignore
4309 // [3] - zero low half of destination
4310 // [5:4] - select 128 bits from sources for high half of destination
4311 // [6] - ignore
4312 // [7] - zero high half of destination
4313
4314 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4315
4316 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4317 unsigned HalfSize = NumElts / 2;
4318 SmallVector<int, 8> ShuffleMask(NumElts);
4319
4320 // Determine which operand(s) are actually in use for this instruction.
4321 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4322 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4323
4324 // If needed, replace operands based on zero mask.
4325 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4326 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4327
4328 // Permute low half of result.
4329 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4330 for (unsigned i = 0; i < HalfSize; ++i)
4331 ShuffleMask[i] = StartIndex + i;
4332
4333 // Permute high half of result.
4334 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4335 for (unsigned i = 0; i < HalfSize; ++i)
4336 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4337
4338 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4339
4340 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4341 Name.starts_with("avx512.mask.vpermil.p") ||
4342 Name.starts_with("avx512.mask.pshuf.d.")) {
4343 Value *Op0 = CI->getArgOperand(0);
4344 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4345 auto *VecTy = cast<FixedVectorType>(CI->getType());
4346 unsigned NumElts = VecTy->getNumElements();
4347 // Calculate the size of each index in the immediate.
4348 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4349 unsigned IdxMask = ((1 << IdxSize) - 1);
4350
4351 SmallVector<int, 8> Idxs(NumElts);
4352 // Lookup the bits for this element, wrapping around the immediate every
4353 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4354 // to offset by the first index of each group.
4355 for (unsigned i = 0; i != NumElts; ++i)
4356 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4357
4358 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4359
4360 if (CI->arg_size() == 4)
4361 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4362 CI->getArgOperand(2));
4363 } else if (Name == "sse2.pshufl.w" ||
4364 Name.starts_with("avx512.mask.pshufl.w.")) {
4365 Value *Op0 = CI->getArgOperand(0);
4366 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4367 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4368
4369 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4370 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4371
4372 SmallVector<int, 16> Idxs(NumElts);
4373 for (unsigned l = 0; l != NumElts; l += 8) {
4374 for (unsigned i = 0; i != 4; ++i)
4375 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4376 for (unsigned i = 4; i != 8; ++i)
4377 Idxs[i + l] = i + l;
4378 }
4379
4380 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4381
4382 if (CI->arg_size() == 4)
4383 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4384 CI->getArgOperand(2));
4385 } else if (Name == "sse2.pshufh.w" ||
4386 Name.starts_with("avx512.mask.pshufh.w.")) {
4387 Value *Op0 = CI->getArgOperand(0);
4388 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4389 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4390
4391 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4392 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4393
4394 SmallVector<int, 16> Idxs(NumElts);
4395 for (unsigned l = 0; l != NumElts; l += 8) {
4396 for (unsigned i = 0; i != 4; ++i)
4397 Idxs[i + l] = i + l;
4398 for (unsigned i = 0; i != 4; ++i)
4399 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4400 }
4401
4402 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4403
4404 if (CI->arg_size() == 4)
4405 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4406 CI->getArgOperand(2));
4407 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4408 Value *Op0 = CI->getArgOperand(0);
4409 Value *Op1 = CI->getArgOperand(1);
4410 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4411 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4412
4413 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4414 unsigned HalfLaneElts = NumLaneElts / 2;
4415
4416 SmallVector<int, 16> Idxs(NumElts);
4417 for (unsigned i = 0; i != NumElts; ++i) {
4418 // Base index is the starting element of the lane.
4419 Idxs[i] = i - (i % NumLaneElts);
4420 // If we are half way through the lane switch to the other source.
4421 if ((i % NumLaneElts) >= HalfLaneElts)
4422 Idxs[i] += NumElts;
4423 // Now select the specific element. By adding HalfLaneElts bits from
4424 // the immediate. Wrapping around the immediate every 8-bits.
4425 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4426 }
4427
4428 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4429
4430 Rep =
4431 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4432 } else if (Name.starts_with("avx512.mask.movddup") ||
4433 Name.starts_with("avx512.mask.movshdup") ||
4434 Name.starts_with("avx512.mask.movsldup")) {
4435 Value *Op0 = CI->getArgOperand(0);
4436 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4437 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4438
4439 unsigned Offset = 0;
4440 if (Name.starts_with("avx512.mask.movshdup."))
4441 Offset = 1;
4442
4443 SmallVector<int, 16> Idxs(NumElts);
4444 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4445 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4446 Idxs[i + l + 0] = i + l + Offset;
4447 Idxs[i + l + 1] = i + l + Offset;
4448 }
4449
4450 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4451
4452 Rep =
4453 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4454 } else if (Name.starts_with("avx512.mask.punpckl") ||
4455 Name.starts_with("avx512.mask.unpckl.")) {
4456 Value *Op0 = CI->getArgOperand(0);
4457 Value *Op1 = CI->getArgOperand(1);
4458 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4459 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4460
4461 SmallVector<int, 64> Idxs(NumElts);
4462 for (int l = 0; l != NumElts; l += NumLaneElts)
4463 for (int i = 0; i != NumLaneElts; ++i)
4464 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4465
4466 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4467
4468 Rep =
4469 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4470 } else if (Name.starts_with("avx512.mask.punpckh") ||
4471 Name.starts_with("avx512.mask.unpckh.")) {
4472 Value *Op0 = CI->getArgOperand(0);
4473 Value *Op1 = CI->getArgOperand(1);
4474 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4475 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4476
4477 SmallVector<int, 64> Idxs(NumElts);
4478 for (int l = 0; l != NumElts; l += NumLaneElts)
4479 for (int i = 0; i != NumLaneElts; ++i)
4480 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4481
4482 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4483
4484 Rep =
4485 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4486 } else if (Name.starts_with("avx512.mask.and.") ||
4487 Name.starts_with("avx512.mask.pand.")) {
4488 VectorType *FTy = cast<VectorType>(CI->getType());
4489 VectorType *ITy = VectorType::getInteger(FTy);
4490 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4491 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4492 Rep = Builder.CreateBitCast(Rep, FTy);
4493 Rep =
4494 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4495 } else if (Name.starts_with("avx512.mask.andn.") ||
4496 Name.starts_with("avx512.mask.pandn.")) {
4497 VectorType *FTy = cast<VectorType>(CI->getType());
4498 VectorType *ITy = VectorType::getInteger(FTy);
4499 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4500 Rep = Builder.CreateAnd(Rep,
4501 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4502 Rep = Builder.CreateBitCast(Rep, FTy);
4503 Rep =
4504 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4505 } else if (Name.starts_with("avx512.mask.or.") ||
4506 Name.starts_with("avx512.mask.por.")) {
4507 VectorType *FTy = cast<VectorType>(CI->getType());
4508 VectorType *ITy = VectorType::getInteger(FTy);
4509 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4510 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4511 Rep = Builder.CreateBitCast(Rep, FTy);
4512 Rep =
4513 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4514 } else if (Name.starts_with("avx512.mask.xor.") ||
4515 Name.starts_with("avx512.mask.pxor.")) {
4516 VectorType *FTy = cast<VectorType>(CI->getType());
4517 VectorType *ITy = VectorType::getInteger(FTy);
4518 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4519 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4520 Rep = Builder.CreateBitCast(Rep, FTy);
4521 Rep =
4522 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4523 } else if (Name.starts_with("avx512.mask.padd.")) {
4524 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4525 Rep =
4526 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4527 } else if (Name.starts_with("avx512.mask.psub.")) {
4528 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4529 Rep =
4530 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4531 } else if (Name.starts_with("avx512.mask.pmull.")) {
4532 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4533 Rep =
4534 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4535 } else if (Name.starts_with("avx512.mask.add.p")) {
4536 if (Name.ends_with(".512")) {
4537 Intrinsic::ID IID;
4538 if (Name[17] == 's')
4539 IID = Intrinsic::x86_avx512_add_ps_512;
4540 else
4541 IID = Intrinsic::x86_avx512_add_pd_512;
4542
4543 Rep = Builder.CreateIntrinsic(
4544 IID,
4545 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4546 } else {
4547 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4548 }
4549 Rep =
4550 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4551 } else if (Name.starts_with("avx512.mask.div.p")) {
4552 if (Name.ends_with(".512")) {
4553 Intrinsic::ID IID;
4554 if (Name[17] == 's')
4555 IID = Intrinsic::x86_avx512_div_ps_512;
4556 else
4557 IID = Intrinsic::x86_avx512_div_pd_512;
4558
4559 Rep = Builder.CreateIntrinsic(
4560 IID,
4561 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4562 } else {
4563 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4564 }
4565 Rep =
4566 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4567 } else if (Name.starts_with("avx512.mask.mul.p")) {
4568 if (Name.ends_with(".512")) {
4569 Intrinsic::ID IID;
4570 if (Name[17] == 's')
4571 IID = Intrinsic::x86_avx512_mul_ps_512;
4572 else
4573 IID = Intrinsic::x86_avx512_mul_pd_512;
4574
4575 Rep = Builder.CreateIntrinsic(
4576 IID,
4577 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4578 } else {
4579 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4580 }
4581 Rep =
4582 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4583 } else if (Name.starts_with("avx512.mask.sub.p")) {
4584 if (Name.ends_with(".512")) {
4585 Intrinsic::ID IID;
4586 if (Name[17] == 's')
4587 IID = Intrinsic::x86_avx512_sub_ps_512;
4588 else
4589 IID = Intrinsic::x86_avx512_sub_pd_512;
4590
4591 Rep = Builder.CreateIntrinsic(
4592 IID,
4593 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4594 } else {
4595 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4596 }
4597 Rep =
4598 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4599 } else if ((Name.starts_with("avx512.mask.max.p") ||
4600 Name.starts_with("avx512.mask.min.p")) &&
4601 Name.drop_front(18) == ".512") {
4602 bool IsDouble = Name[17] == 'd';
4603 bool IsMin = Name[13] == 'i';
4604 static const Intrinsic::ID MinMaxTbl[2][2] = {
4605 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4606 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4607 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4608
4609 Rep = Builder.CreateIntrinsic(
4610 IID,
4611 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4612 Rep =
4613 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4614 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4615 Rep =
4616 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4617 {CI->getArgOperand(0), Builder.getInt1(false)});
4618 Rep =
4619 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4620 } else if (Name.starts_with("avx512.mask.psll")) {
4621 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4622 bool IsVariable = Name[16] == 'v';
4623 char Size = Name[16] == '.' ? Name[17]
4624 : Name[17] == '.' ? Name[18]
4625 : Name[18] == '.' ? Name[19]
4626 : Name[20];
4627
4628 Intrinsic::ID IID;
4629 if (IsVariable && Name[17] != '.') {
4630 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4631 IID = Intrinsic::x86_avx2_psllv_q;
4632 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4633 IID = Intrinsic::x86_avx2_psllv_q_256;
4634 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4635 IID = Intrinsic::x86_avx2_psllv_d;
4636 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4637 IID = Intrinsic::x86_avx2_psllv_d_256;
4638 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4639 IID = Intrinsic::x86_avx512_psllv_w_128;
4640 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4641 IID = Intrinsic::x86_avx512_psllv_w_256;
4642 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4643 IID = Intrinsic::x86_avx512_psllv_w_512;
4644 else
4645 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4646 } else if (Name.ends_with(".128")) {
4647 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4648 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4649 : Intrinsic::x86_sse2_psll_d;
4650 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4651 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4652 : Intrinsic::x86_sse2_psll_q;
4653 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4654 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4655 : Intrinsic::x86_sse2_psll_w;
4656 else
4657 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4658 } else if (Name.ends_with(".256")) {
4659 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4660 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4661 : Intrinsic::x86_avx2_psll_d;
4662 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4663 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4664 : Intrinsic::x86_avx2_psll_q;
4665 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4666 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4667 : Intrinsic::x86_avx2_psll_w;
4668 else
4669 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4670 } else {
4671 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4672 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4673 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4674 : Intrinsic::x86_avx512_psll_d_512;
4675 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4676 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4677 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4678 : Intrinsic::x86_avx512_psll_q_512;
4679 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4680 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4681 : Intrinsic::x86_avx512_psll_w_512;
4682 else
4683 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4684 }
4685
4686 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4687 } else if (Name.starts_with("avx512.mask.psrl")) {
4688 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4689 bool IsVariable = Name[16] == 'v';
4690 char Size = Name[16] == '.' ? Name[17]
4691 : Name[17] == '.' ? Name[18]
4692 : Name[18] == '.' ? Name[19]
4693 : Name[20];
4694
4695 Intrinsic::ID IID;
4696 if (IsVariable && Name[17] != '.') {
4697 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4698 IID = Intrinsic::x86_avx2_psrlv_q;
4699 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4700 IID = Intrinsic::x86_avx2_psrlv_q_256;
4701 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4702 IID = Intrinsic::x86_avx2_psrlv_d;
4703 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4704 IID = Intrinsic::x86_avx2_psrlv_d_256;
4705 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4706 IID = Intrinsic::x86_avx512_psrlv_w_128;
4707 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4708 IID = Intrinsic::x86_avx512_psrlv_w_256;
4709 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4710 IID = Intrinsic::x86_avx512_psrlv_w_512;
4711 else
4712 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4713 } else if (Name.ends_with(".128")) {
4714 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4715 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4716 : Intrinsic::x86_sse2_psrl_d;
4717 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4718 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4719 : Intrinsic::x86_sse2_psrl_q;
4720 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4721 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4722 : Intrinsic::x86_sse2_psrl_w;
4723 else
4724 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4725 } else if (Name.ends_with(".256")) {
4726 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4727 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4728 : Intrinsic::x86_avx2_psrl_d;
4729 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4730 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4731 : Intrinsic::x86_avx2_psrl_q;
4732 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4733 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4734 : Intrinsic::x86_avx2_psrl_w;
4735 else
4736 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4737 } else {
4738 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4739 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4740 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4741 : Intrinsic::x86_avx512_psrl_d_512;
4742 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4743 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4744 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4745 : Intrinsic::x86_avx512_psrl_q_512;
4746 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4747 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4748 : Intrinsic::x86_avx512_psrl_w_512;
4749 else
4750 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4751 }
4752
4753 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4754 } else if (Name.starts_with("avx512.mask.psra")) {
4755 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4756 bool IsVariable = Name[16] == 'v';
4757 char Size = Name[16] == '.' ? Name[17]
4758 : Name[17] == '.' ? Name[18]
4759 : Name[18] == '.' ? Name[19]
4760 : Name[20];
4761
4762 Intrinsic::ID IID;
4763 if (IsVariable && Name[17] != '.') {
4764 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4765 IID = Intrinsic::x86_avx2_psrav_d;
4766 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4767 IID = Intrinsic::x86_avx2_psrav_d_256;
4768 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4769 IID = Intrinsic::x86_avx512_psrav_w_128;
4770 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4771 IID = Intrinsic::x86_avx512_psrav_w_256;
4772 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4773 IID = Intrinsic::x86_avx512_psrav_w_512;
4774 else
4775 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4776 } else if (Name.ends_with(".128")) {
4777 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4778 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4779 : Intrinsic::x86_sse2_psra_d;
4780 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4781 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4782 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4783 : Intrinsic::x86_avx512_psra_q_128;
4784 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4785 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4786 : Intrinsic::x86_sse2_psra_w;
4787 else
4788 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4789 } else if (Name.ends_with(".256")) {
4790 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4791 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4792 : Intrinsic::x86_avx2_psra_d;
4793 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4794 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4795 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4796 : Intrinsic::x86_avx512_psra_q_256;
4797 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4798 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4799 : Intrinsic::x86_avx2_psra_w;
4800 else
4801 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4802 } else {
4803 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4804 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4805 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4806 : Intrinsic::x86_avx512_psra_d_512;
4807 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4808 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4809 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4810 : Intrinsic::x86_avx512_psra_q_512;
4811 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4812 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4813 : Intrinsic::x86_avx512_psra_w_512;
4814 else
4815 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4816 }
4817
4818 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4819 } else if (Name.starts_with("avx512.mask.move.s")) {
4820 Rep = upgradeMaskedMove(Builder, *CI);
4821 } else if (Name.starts_with("avx512.cvtmask2")) {
4822 Rep = upgradeMaskToInt(Builder, *CI);
4823 } else if (Name.ends_with(".movntdqa")) {
4824 MDNode *Node = MDNode::get(
4825 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4826
4827 LoadInst *LI = Builder.CreateAlignedLoad(
4828 CI->getType(), CI->getArgOperand(0),
4829 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4830 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4831 Rep = LI;
4832 } else if (Name.starts_with("fma.vfmadd.") ||
4833 Name.starts_with("fma.vfmsub.") ||
4834 Name.starts_with("fma.vfnmadd.") ||
4835 Name.starts_with("fma.vfnmsub.")) {
4836 bool NegMul = Name[6] == 'n';
4837 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4838 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4839
4840 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4841 CI->getArgOperand(2)};
4842
4843 if (IsScalar) {
4844 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4845 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4846 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4847 }
4848
4849 if (NegMul && !IsScalar)
4850 Ops[0] = Builder.CreateFNeg(Ops[0]);
4851 if (NegMul && IsScalar)
4852 Ops[1] = Builder.CreateFNeg(Ops[1]);
4853 if (NegAcc)
4854 Ops[2] = Builder.CreateFNeg(Ops[2]);
4855
4856 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4857
4858 if (IsScalar)
4859 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4860 } else if (Name.starts_with("fma4.vfmadd.s")) {
4861 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4862 CI->getArgOperand(2)};
4863
4864 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4865 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4866 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4867
4868 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4869
4870 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4871 Rep, (uint64_t)0);
4872 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4873 Name.starts_with("avx512.maskz.vfmadd.s") ||
4874 Name.starts_with("avx512.mask3.vfmadd.s") ||
4875 Name.starts_with("avx512.mask3.vfmsub.s") ||
4876 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4877 bool IsMask3 = Name[11] == '3';
4878 bool IsMaskZ = Name[11] == 'z';
4879 // Drop the "avx512.mask." to make it easier.
4880 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4881 bool NegMul = Name[2] == 'n';
4882 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4883
4884 Value *A = CI->getArgOperand(0);
4885 Value *B = CI->getArgOperand(1);
4886 Value *C = CI->getArgOperand(2);
4887
4888 if (NegMul && (IsMask3 || IsMaskZ))
4889 A = Builder.CreateFNeg(A);
4890 if (NegMul && !(IsMask3 || IsMaskZ))
4891 B = Builder.CreateFNeg(B);
4892 if (NegAcc)
4893 C = Builder.CreateFNeg(C);
4894
4895 A = Builder.CreateExtractElement(A, (uint64_t)0);
4896 B = Builder.CreateExtractElement(B, (uint64_t)0);
4897 C = Builder.CreateExtractElement(C, (uint64_t)0);
4898
4899 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4900 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4901 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4902
4903 Intrinsic::ID IID;
4904 if (Name.back() == 'd')
4905 IID = Intrinsic::x86_avx512_vfmadd_f64;
4906 else
4907 IID = Intrinsic::x86_avx512_vfmadd_f32;
4908 Rep = Builder.CreateIntrinsic(IID, Ops);
4909 } else {
4910 Rep = Builder.CreateFMA(A, B, C);
4911 }
4912
4913 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4914 : IsMask3 ? C
4915 : A;
4916
4917 // For Mask3 with NegAcc, we need to create a new extractelement that
4918 // avoids the negation above.
4919 if (NegAcc && IsMask3)
4920 PassThru =
4921 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4922
4923 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4924 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4925 (uint64_t)0);
4926 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4927 Name.starts_with("avx512.mask.vfnmadd.p") ||
4928 Name.starts_with("avx512.mask.vfnmsub.p") ||
4929 Name.starts_with("avx512.mask3.vfmadd.p") ||
4930 Name.starts_with("avx512.mask3.vfmsub.p") ||
4931 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4932 Name.starts_with("avx512.maskz.vfmadd.p")) {
4933 bool IsMask3 = Name[11] == '3';
4934 bool IsMaskZ = Name[11] == 'z';
4935 // Drop the "avx512.mask." to make it easier.
4936 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4937 bool NegMul = Name[2] == 'n';
4938 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4939
4940 Value *A = CI->getArgOperand(0);
4941 Value *B = CI->getArgOperand(1);
4942 Value *C = CI->getArgOperand(2);
4943
4944 if (NegMul && (IsMask3 || IsMaskZ))
4945 A = Builder.CreateFNeg(A);
4946 if (NegMul && !(IsMask3 || IsMaskZ))
4947 B = Builder.CreateFNeg(B);
4948 if (NegAcc)
4949 C = Builder.CreateFNeg(C);
4950
4951 if (CI->arg_size() == 5 &&
4952 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4953 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4954 Intrinsic::ID IID;
4955 // Check the character before ".512" in string.
4956 if (Name[Name.size() - 5] == 's')
4957 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4958 else
4959 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4960
4961 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4962 } else {
4963 Rep = Builder.CreateFMA(A, B, C);
4964 }
4965
4966 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4967 : IsMask3 ? CI->getArgOperand(2)
4968 : CI->getArgOperand(0);
4969
4970 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4971 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4972 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4973 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4974 Intrinsic::ID IID;
4975 if (VecWidth == 128 && EltWidth == 32)
4976 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4977 else if (VecWidth == 256 && EltWidth == 32)
4978 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4979 else if (VecWidth == 128 && EltWidth == 64)
4980 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4981 else if (VecWidth == 256 && EltWidth == 64)
4982 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4983 else
4984 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4985
4986 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4987 CI->getArgOperand(2)};
4988 Ops[2] = Builder.CreateFNeg(Ops[2]);
4989 Rep = Builder.CreateIntrinsic(IID, Ops);
4990 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4991 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4992 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4993 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4994 bool IsMask3 = Name[11] == '3';
4995 bool IsMaskZ = Name[11] == 'z';
4996 // Drop the "avx512.mask." to make it easier.
4997 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4998 bool IsSubAdd = Name[3] == 's';
4999 if (CI->arg_size() == 5) {
5000 Intrinsic::ID IID;
5001 // Check the character before ".512" in string.
5002 if (Name[Name.size() - 5] == 's')
5003 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
5004 else
5005 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
5006
5007 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5008 CI->getArgOperand(2), CI->getArgOperand(4)};
5009 if (IsSubAdd)
5010 Ops[2] = Builder.CreateFNeg(Ops[2]);
5011
5012 Rep = Builder.CreateIntrinsic(IID, Ops);
5013 } else {
5014 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
5015
5016 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5017 CI->getArgOperand(2)};
5018
5019 Function *FMA = Intrinsic::getOrInsertDeclaration(
5020 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
5021 Value *Odd = Builder.CreateCall(FMA, Ops);
5022 Ops[2] = Builder.CreateFNeg(Ops[2]);
5023 Value *Even = Builder.CreateCall(FMA, Ops);
5024
5025 if (IsSubAdd)
5026 std::swap(Even, Odd);
5027
5028 SmallVector<int, 32> Idxs(NumElts);
5029 for (int i = 0; i != NumElts; ++i)
5030 Idxs[i] = i + (i % 2) * NumElts;
5031
5032 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
5033 }
5034
5035 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
5036 : IsMask3 ? CI->getArgOperand(2)
5037 : CI->getArgOperand(0);
5038
5039 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5040 } else if (Name.starts_with("avx512.mask.pternlog.") ||
5041 Name.starts_with("avx512.maskz.pternlog.")) {
5042 bool ZeroMask = Name[11] == 'z';
5043 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5044 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
5045 Intrinsic::ID IID;
5046 if (VecWidth == 128 && EltWidth == 32)
5047 IID = Intrinsic::x86_avx512_pternlog_d_128;
5048 else if (VecWidth == 256 && EltWidth == 32)
5049 IID = Intrinsic::x86_avx512_pternlog_d_256;
5050 else if (VecWidth == 512 && EltWidth == 32)
5051 IID = Intrinsic::x86_avx512_pternlog_d_512;
5052 else if (VecWidth == 128 && EltWidth == 64)
5053 IID = Intrinsic::x86_avx512_pternlog_q_128;
5054 else if (VecWidth == 256 && EltWidth == 64)
5055 IID = Intrinsic::x86_avx512_pternlog_q_256;
5056 else if (VecWidth == 512 && EltWidth == 64)
5057 IID = Intrinsic::x86_avx512_pternlog_q_512;
5058 else
5059 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5060
5061 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5062 CI->getArgOperand(2), CI->getArgOperand(3)};
5063 Rep = Builder.CreateIntrinsic(IID, Args);
5064 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5065 : CI->getArgOperand(0);
5066 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
5067 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
5068 Name.starts_with("avx512.maskz.vpmadd52")) {
5069 bool ZeroMask = Name[11] == 'z';
5070 bool High = Name[20] == 'h' || Name[21] == 'h';
5071 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5072 Intrinsic::ID IID;
5073 if (VecWidth == 128 && !High)
5074 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5075 else if (VecWidth == 256 && !High)
5076 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5077 else if (VecWidth == 512 && !High)
5078 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5079 else if (VecWidth == 128 && High)
5080 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5081 else if (VecWidth == 256 && High)
5082 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5083 else if (VecWidth == 512 && High)
5084 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5085 else
5086 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5087
5088 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5089 CI->getArgOperand(2)};
5090 Rep = Builder.CreateIntrinsic(IID, Args);
5091 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5092 : CI->getArgOperand(0);
5093 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5094 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5095 Name.starts_with("avx512.mask.vpermt2var.") ||
5096 Name.starts_with("avx512.maskz.vpermt2var.")) {
5097 bool ZeroMask = Name[11] == 'z';
5098 bool IndexForm = Name[17] == 'i';
5099 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5100 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5101 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5102 Name.starts_with("avx512.mask.vpdpbusds.") ||
5103 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5104 bool ZeroMask = Name[11] == 'z';
5105 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5106 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5107 Intrinsic::ID IID;
5108 if (VecWidth == 128 && !IsSaturating)
5109 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5110 else if (VecWidth == 256 && !IsSaturating)
5111 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5112 else if (VecWidth == 512 && !IsSaturating)
5113 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5114 else if (VecWidth == 128 && IsSaturating)
5115 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5116 else if (VecWidth == 256 && IsSaturating)
5117 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5118 else if (VecWidth == 512 && IsSaturating)
5119 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5120 else
5121 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5122
5123 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5124 CI->getArgOperand(2)};
5125
5126 // Input arguments types were incorrectly set to vectors of i32 before but
5127 // they should be vectors of i8. Insert bit cast when encountering the old
5128 // types
5129 if (Args[1]->getType()->isVectorTy() &&
5130 cast<VectorType>(Args[1]->getType())
5131 ->getElementType()
5132 ->isIntegerTy(32) &&
5133 Args[2]->getType()->isVectorTy() &&
5134 cast<VectorType>(Args[2]->getType())
5135 ->getElementType()
5136 ->isIntegerTy(32)) {
5137 Type *NewArgType = nullptr;
5138 if (VecWidth == 128)
5139 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5140 else if (VecWidth == 256)
5141 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5142 else if (VecWidth == 512)
5143 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5144 else
5145 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5146 CI);
5147
5148 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5149 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5150 }
5151
5152 Rep = Builder.CreateIntrinsic(IID, Args);
5153 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5154 : CI->getArgOperand(0);
5155 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5156 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5157 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5158 Name.starts_with("avx512.mask.vpdpwssds.") ||
5159 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5160 bool ZeroMask = Name[11] == 'z';
5161 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5162 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5163 Intrinsic::ID IID;
5164 if (VecWidth == 128 && !IsSaturating)
5165 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5166 else if (VecWidth == 256 && !IsSaturating)
5167 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5168 else if (VecWidth == 512 && !IsSaturating)
5169 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5170 else if (VecWidth == 128 && IsSaturating)
5171 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5172 else if (VecWidth == 256 && IsSaturating)
5173 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5174 else if (VecWidth == 512 && IsSaturating)
5175 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5176 else
5177 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5178
5179 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5180 CI->getArgOperand(2)};
5181
5182 // Input arguments types were incorrectly set to vectors of i32 before but
5183 // they should be vectors of i16. Insert bit cast when encountering the old
5184 // types
5185 if (Args[1]->getType()->isVectorTy() &&
5186 cast<VectorType>(Args[1]->getType())
5187 ->getElementType()
5188 ->isIntegerTy(32) &&
5189 Args[2]->getType()->isVectorTy() &&
5190 cast<VectorType>(Args[2]->getType())
5191 ->getElementType()
5192 ->isIntegerTy(32)) {
5193 Type *NewArgType = nullptr;
5194 if (VecWidth == 128)
5195 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5196 else if (VecWidth == 256)
5197 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5198 else if (VecWidth == 512)
5199 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5200 else
5201 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5202 CI);
5203
5204 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5205 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5206 }
5207
5208 Rep = Builder.CreateIntrinsic(IID, Args);
5209 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5210 : CI->getArgOperand(0);
5211 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5212 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5213 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5214 Name == "subborrow.u32" || Name == "subborrow.u64") {
5215 Intrinsic::ID IID;
5216 if (Name[0] == 'a' && Name.back() == '2')
5217 IID = Intrinsic::x86_addcarry_32;
5218 else if (Name[0] == 'a' && Name.back() == '4')
5219 IID = Intrinsic::x86_addcarry_64;
5220 else if (Name[0] == 's' && Name.back() == '2')
5221 IID = Intrinsic::x86_subborrow_32;
5222 else if (Name[0] == 's' && Name.back() == '4')
5223 IID = Intrinsic::x86_subborrow_64;
5224 else
5225 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5226
5227 // Make a call with 3 operands.
5228 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5229 CI->getArgOperand(2)};
5230 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5231
5232 // Extract the second result and store it.
5233 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5234 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5235 // Replace the original call result with the first result of the new call.
5236 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5237
5238 CI->replaceAllUsesWith(CF);
5239 Rep = nullptr;
5240 } else if (Name.starts_with("avx512.mask.") &&
5241 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5242 // Rep will be updated by the call in the condition.
5243 } else if (Name.starts_with("bmi.pdep.")) {
5244 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5245 } else if (Name.starts_with("bmi.pext.")) {
5246 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5247 } else
5248 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5249
5250 return Rep;
5251}
5252
5253static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5254 Function *F, IRBuilder<> &Builder) {
5255 if (Name.starts_with("neon.bfcvt")) {
5256 if (Name.starts_with("neon.bfcvtn2")) {
5257 SmallVector<int, 32> LoMask(4);
5258 std::iota(LoMask.begin(), LoMask.end(), 0);
5259 SmallVector<int, 32> ConcatMask(8);
5260 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5261 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5262 Value *Trunc =
5263 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5264 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5265 } else if (Name.starts_with("neon.bfcvtn")) {
5266 SmallVector<int, 32> ConcatMask(8);
5267 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5268 Type *V4BF16 =
5269 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5270 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5271 dbgs() << "Trunc: " << *Trunc << "\n";
5272 return Builder.CreateShuffleVector(
5273 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5274 } else {
5275 return Builder.CreateFPTrunc(CI->getOperand(0),
5276 Type::getBFloatTy(F->getContext()));
5277 }
5278 } else if (Name.starts_with("sve.fcvt")) {
5279 Intrinsic::ID NewID =
5280 StringSwitch<Intrinsic::ID>(Name)
5281 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5282 .Case("sve.fcvtnt.bf16f32",
5283 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5284 .Default(Intrinsic::not_intrinsic);
5285 if (NewID == Intrinsic::not_intrinsic)
5286 llvm_unreachable("Unhandled Intrinsic!");
5287
5288 SmallVector<Value *, 3> Args(CI->args());
5289
5290 // The original intrinsics incorrectly used a predicate based on the
5291 // smallest element type rather than the largest.
5292 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5293 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5294
5295 if (Args[1]->getType() != BadPredTy)
5296 llvm_unreachable("Unexpected predicate type!");
5297
5298 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5299 BadPredTy, Args[1]);
5300 Args[1] = Builder.CreateIntrinsic(
5301 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5302
5303 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5304 CI->getName());
5305 }
5306
5307 if (Name == "neon.vcvtfp2hf")
5308 return Builder.CreateBitCast(
5309 Builder.CreateFPTrunc(
5310 CI->getOperand(0),
5311 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5312 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5313 if (Name == "neon.vcvthf2fp")
5314 return Builder.CreateFPExt(
5315 Builder.CreateBitCast(
5316 CI->getOperand(0),
5317 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5318 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5319
5320 llvm_unreachable("Unhandled Intrinsic!");
5321}
5322
5323static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5324 IRBuilder<> &Builder) {
5325 if (Name == "mve.vctp64.old") {
5326 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5327 // correct type.
5328 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5329 CI->getArgOperand(0),
5330 /*FMFSource=*/nullptr, CI->getName());
5331 Value *C1 = Builder.CreateIntrinsic(
5332 Intrinsic::arm_mve_pred_v2i,
5333 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5334 return Builder.CreateIntrinsic(
5335 Intrinsic::arm_mve_pred_i2v,
5336 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5337 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5338 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5339 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5340 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5341 Name ==
5342 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5343 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5344 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5345 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5346 Name ==
5347 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5348 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5349 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5350 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5351 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5352 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5353 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5354 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5355 std::vector<Type *> Tys;
5356 unsigned ID = CI->getIntrinsicID();
5357 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5358 switch (ID) {
5359 case Intrinsic::arm_mve_mull_int_predicated:
5360 case Intrinsic::arm_mve_vqdmull_predicated:
5361 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5362 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5363 break;
5364 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5365 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5366 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5367 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5368 V2I1Ty};
5369 break;
5370 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5371 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5372 CI->getOperand(1)->getType(), V2I1Ty};
5373 break;
5374 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5375 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5376 CI->getOperand(2)->getType(), V2I1Ty};
5377 break;
5378 case Intrinsic::arm_cde_vcx1q_predicated:
5379 case Intrinsic::arm_cde_vcx1qa_predicated:
5380 case Intrinsic::arm_cde_vcx2q_predicated:
5381 case Intrinsic::arm_cde_vcx2qa_predicated:
5382 case Intrinsic::arm_cde_vcx3q_predicated:
5383 case Intrinsic::arm_cde_vcx3qa_predicated:
5384 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5385 break;
5386 default:
5387 llvm_unreachable("Unhandled Intrinsic!");
5388 }
5389
5390 std::vector<Value *> Ops;
5391 for (Value *Op : CI->args()) {
5392 Type *Ty = Op->getType();
5393 if (Ty->getScalarSizeInBits() == 1) {
5394 Value *C1 = Builder.CreateIntrinsic(
5395 Intrinsic::arm_mve_pred_v2i,
5396 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5397 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5398 }
5399 Ops.push_back(Op);
5400 }
5401
5402 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5403 CI->getName());
5404 }
5405 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5406}
5407
5408// These are expected to have the arguments:
5409// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5410//
5411// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5412//
5413static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5414 Function *F, IRBuilder<> &Builder) {
5415 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5416 // for compatibility.
5417 auto UpgradeLegacyWMMAIUIntrinsicCall =
5418 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5419 ArrayRef<Type *> OverloadTys) -> Value * {
5420 // Prepare arguments, append clamp=0 for compatibility
5421 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5422 Args.push_back(Builder.getFalse());
5423
5424 // Insert the declaration for the right overload types
5425 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5426 F->getParent(), F->getIntrinsicID(), OverloadTys);
5427
5428 // Copy operand bundles if any
5429 SmallVector<OperandBundleDef, 1> Bundles;
5430 CI->getOperandBundlesAsDefs(Bundles);
5431
5432 // Create the new call and copy calling properties
5433 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5434 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5435 NewCall->setCallingConv(CI->getCallingConv());
5436 NewCall->setAttributes(CI->getAttributes());
5437 NewCall->copyMetadata(*CI);
5438 return NewCall;
5439 };
5440
5441 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5442 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5443 "intrinsic should have 7 arguments");
5444 Type *T1 = CI->getArgOperand(4)->getType();
5445 Type *T2 = CI->getArgOperand(1)->getType();
5446 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5447 }
5448 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5449 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5450 "intrinsic should have 8 arguments");
5451 Type *T1 = CI->getArgOperand(4)->getType();
5452 Type *T2 = CI->getArgOperand(1)->getType();
5453 Type *T3 = CI->getArgOperand(3)->getType();
5454 Type *T4 = CI->getArgOperand(5)->getType();
5455 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5456 }
5457
5458 switch (F->getIntrinsicID()) {
5459 default:
5460 break;
5461 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5462 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5463 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5464 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5465 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5466 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5467 // Drop src0 and src1 modifiers.
5468 const Value *Op0 = CI->getArgOperand(0);
5469 const Value *Op2 = CI->getArgOperand(2);
5470 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5471 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5472 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5473 if (!ModA->isZero() || !ModB->isZero())
5474 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5475
5476 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5477 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5478 Args.push_back(CI->getArgOperand(I));
5479
5480 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5481 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5482 Overloads.push_back(Args[3]->getType());
5483 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5484 F->getParent(), F->getIntrinsicID(), Overloads);
5485
5486 SmallVector<OperandBundleDef, 1> Bundles;
5487 CI->getOperandBundlesAsDefs(Bundles);
5488
5489 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5490 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5491 NewCall->setCallingConv(CI->getCallingConv());
5492 NewCall->setAttributes(CI->getAttributes());
5493 NewCall->copyMetadata(*CI);
5494 NewCall->takeName(CI);
5495 return NewCall;
5496 }
5497 }
5498
5499 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5500 Value *LHS = CI->getArgOperand(0);
5501 Value *RHS = CI->getArgOperand(1);
5502 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5503 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5504 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5505 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5506 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5507 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5508 NewCall->setCallingConv(CI->getCallingConv());
5509 NewCall->copyMetadata(*CI);
5510 NewCall->takeName(CI);
5511 return NewCall;
5512 }
5513
5514 if (Name.starts_with("addrspacecast.nonnull")) {
5515 if (CI->getNumOperands() < 2) // Malformed bitcode.
5516 return nullptr;
5517 Value *ASC = Builder.CreateAddrSpaceCast(
5518 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5519 ASC->takeName(CI);
5520 return ASC;
5521 }
5522
5523 AtomicRMWInst::BinOp RMWOp =
5524 StringSwitch<AtomicRMWInst::BinOp>(Name)
5525 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5526 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5527 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5528 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5529 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5530 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5531 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5532 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5533 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5534 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5535 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5536 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5537 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5538
5539 unsigned NumOperands = CI->getNumOperands();
5540 if (NumOperands < 3) // Malformed bitcode.
5541 return nullptr;
5542
5543 Value *Ptr = CI->getArgOperand(0);
5544 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5545 if (!PtrTy) // Malformed.
5546 return nullptr;
5547
5548 Value *Val = CI->getArgOperand(1);
5549 if (Val->getType() != CI->getType()) // Malformed.
5550 return nullptr;
5551
5552 ConstantInt *OrderArg = nullptr;
5553 bool IsVolatile = false;
5554
5555 // These should have 5 arguments (plus the callee). A separate version of the
5556 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5557 if (NumOperands > 3)
5558 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5559
5560 // Ignore scope argument at 3
5561
5562 if (NumOperands > 5) {
5563 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5564 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5565 }
5566
5567 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5568 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5569 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5570 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5571 Order = AtomicOrdering::SequentiallyConsistent;
5572
5573 LLVMContext &Ctx = F->getContext();
5574
5575 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5576 Type *RetTy = CI->getType();
5577 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5578 if (VT->getElementType()->isIntegerTy(16)) {
5579 VectorType *AsBF16 =
5580 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5581 Val = Builder.CreateBitCast(Val, AsBF16);
5582 }
5583 }
5584
5585 // The scope argument never really worked correctly. Use agent as the most
5586 // conservative option which should still always produce the instruction.
5587 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5588 AtomicRMWInst *RMW =
5589 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5590
5591 unsigned AddrSpace = PtrTy->getAddressSpace();
5592 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5593 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5594 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5595 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5596 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5597 }
5598
5599 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5600 MDBuilder MDB(F->getContext());
5601 MDNode *RangeNotPrivate =
5602 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5603 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5604 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5605 }
5606
5607 if (IsVolatile)
5608 RMW->setVolatile(true);
5609
5610 return Builder.CreateBitCast(RMW, RetTy);
5611}
5612
5613/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5614/// plain MDNode, as it's the verifier's job to check these are the correct
5615/// types later.
5616static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5617 if (Op < CI->arg_size()) {
5618 if (MetadataAsValue *MAV =
5619 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5620 Metadata *MD = MAV->getMetadata();
5621 return dyn_cast_if_present<MDNode>(MD);
5622 }
5623 }
5624 return nullptr;
5625}
5626
5627/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5628static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5629 if (Op < CI->arg_size())
5630 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5631 return MAV->getMetadata();
5632 return nullptr;
5633}
5634
5635/// Convert debug intrinsic calls to non-instruction debug records.
5636/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5637/// \p CI - The debug intrinsic call.
5638static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5639 DbgRecord *DR = nullptr;
5640 if (Name == "label") {
5641 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5642 } else if (Name == "assign") {
5643 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5644 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5645 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5646 unwrapMAVMetadataOp(CI, 4),
5647 /*The address is a Value ref, it will be stored as a Metadata */
5648 unwrapMAVOp(CI, 5));
5649 } else if (Name == "declare") {
5650 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5651 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5652 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5653 } else if (Name == "addr") {
5654 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5655 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5656 // Don't try to add something to the expression if it's not an expression.
5657 // Instead, allow the verifier to fail later.
5658 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5659 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5660 }
5661 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5662 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5663 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5664 } else if (Name == "value") {
5665 // An old version of dbg.value had an extra offset argument.
5666 unsigned VarOp = 1;
5667 unsigned ExprOp = 2;
5668 if (CI->arg_size() == 4) {
5669 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5670 // Nonzero offset dbg.values get dropped without a replacement.
5671 if (!Offset || !Offset->isNullValue())
5672 return;
5673 VarOp = 2;
5674 ExprOp = 3;
5675 }
5676 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5677 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5678 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5679 nullptr);
5680 }
5681 DR->setDebugLoc(CI->getDebugLoc());
5682 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5683 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5684}
5685
5686static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5687 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5688 if (!Offset)
5689 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5690 int64_t OffsetVal = Offset->getSExtValue();
5691 return Builder.CreateIntrinsic(OffsetVal >= 0
5692 ? Intrinsic::vector_splice_left
5693 : Intrinsic::vector_splice_right,
5694 CI->getType(),
5695 {CI->getArgOperand(0), CI->getArgOperand(1),
5696 Builder.getInt32(std::abs(OffsetVal))});
5697}
5698
5699static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5700 Function *F, IRBuilder<> &Builder) {
5701 if (Name.starts_with("to.fp16")) {
5702 Value *Cast =
5703 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5704 return Builder.CreateBitCast(Cast, CI->getType());
5705 }
5706
5707 if (Name.starts_with("from.fp16")) {
5708 Value *Cast =
5709 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5710 return Builder.CreateFPExt(Cast, CI->getType());
5711 }
5712
5713 return nullptr;
5714}
5715
5716static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5717 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5718 if (!MD || !isa<MDString>(MD))
5719 return ICmpInst::BAD_ICMP_PREDICATE;
5720 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5721 .Case("eq", ICmpInst::ICMP_EQ)
5722 .Case("ne", ICmpInst::ICMP_NE)
5723 .Case("ugt", ICmpInst::ICMP_UGT)
5724 .Case("uge", ICmpInst::ICMP_UGE)
5725 .Case("ult", ICmpInst::ICMP_ULT)
5726 .Case("ule", ICmpInst::ICMP_ULE)
5727 .Case("sgt", ICmpInst::ICMP_SGT)
5728 .Case("sge", ICmpInst::ICMP_SGE)
5729 .Case("slt", ICmpInst::ICMP_SLT)
5730 .Case("sle", ICmpInst::ICMP_SLE)
5731 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5732}
5733
5734static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5735 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5736 if (!MD || !isa<MDString>(MD))
5737 return FCmpInst::BAD_FCMP_PREDICATE;
5738 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5739 .Case("oeq", FCmpInst::FCMP_OEQ)
5740 .Case("ogt", FCmpInst::FCMP_OGT)
5741 .Case("oge", FCmpInst::FCMP_OGE)
5742 .Case("olt", FCmpInst::FCMP_OLT)
5743 .Case("ole", FCmpInst::FCMP_OLE)
5744 .Case("one", FCmpInst::FCMP_ONE)
5745 .Case("ord", FCmpInst::FCMP_ORD)
5746 .Case("uno", FCmpInst::FCMP_UNO)
5747 .Case("ueq", FCmpInst::FCMP_UEQ)
5748 .Case("ugt", FCmpInst::FCMP_UGT)
5749 .Case("uge", FCmpInst::FCMP_UGE)
5750 .Case("ult", FCmpInst::FCMP_ULT)
5751 .Case("ule", FCmpInst::FCMP_ULE)
5752 .Case("une", FCmpInst::FCMP_UNE)
5753 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5754}
5755
5756static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5757 IRBuilder<> &Builder) {
5758 Value *Rep;
5759 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5760 if (Opcode && Instruction::isUnaryOp(Opcode))
5761 Rep =
5762 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5763 else if (Opcode && Instruction::isBinaryOp(Opcode))
5764 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5765 CI->getArgOperand(0), CI->getArgOperand(1));
5766 else if (Opcode && Instruction::isCast(Opcode))
5767 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5768 CI->getType());
5769 else if (Opcode == Instruction::ICmp)
5770 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5771 CI->getArgOperand(0), CI->getArgOperand(1));
5772 else if (Opcode == Instruction::FCmp)
5773 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5774 CI->getArgOperand(0), CI->getArgOperand(1));
5775 else if (Opcode == Instruction::Select)
5776 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5777 CI->getArgOperand(2));
5778 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5779 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5780 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5781 } else
5782 llvm_unreachable("Unexpected vp intrinsic");
5783 Rep->takeName(CI);
5784 return Rep;
5785}
5786
5787static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5788 IRBuilder<> &Builder) {
5789 Intrinsic::ID IID = NewFn->getIntrinsicID();
5790
5791 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5792 if (Defaults.empty())
5793 return false;
5794
5795 unsigned OldArgCount = CI->arg_size();
5796 unsigned NewArgCount = NewFn->arg_size();
5797
5798 if (OldArgCount < FirstDefault)
5799 return false;
5800
5801 // More arguments than the new intrinsic accepts, cannot upgrade.
5802 if (OldArgCount > NewArgCount)
5803 return false;
5804
5805 // The call already passes the defaulted arguments explicitly; only the
5806 // callee is still the old, shorter declaration, so retarget it.
5807 if (OldArgCount == NewArgCount) {
5808 if (CI->getFunctionType() != NewFn->getFunctionType())
5809 return false;
5810 CI->setCalledFunction(NewFn);
5811 return true;
5812 }
5813
5814 // OldArgCount < NewArgCount: Fill in each missing trailing default
5815 // argument from the table.
5816 SmallVector<Value *, 8> NewArgs(CI->args());
5817
5818 FunctionType *NewFT = NewFn->getFunctionType();
5819 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5820 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5821 "missing argument outside the default range");
5822 Type *ParamTy = NewFT->getParamType(Idx);
5823
5824 // Only integer types are supported (i1, i8, i16, i32, i64).
5825 if (!ParamTy->isIntegerTy())
5826 return false;
5827 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5828 }
5829
5830 // Preserve operand bundles by creating the call with them.
5831 SmallVector<OperandBundleDef, 1> OpBundles;
5832 CI->getOperandBundlesAsDefs(OpBundles);
5833 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5834
5835 NewCall->takeName(CI);
5836 NewCall->setCallingConv(CI->getCallingConv());
5837 NewCall->copyMetadata(*CI);
5838 if (auto *OldCI = dyn_cast<CallInst>(CI))
5839 NewCall->setTailCallKind(OldCI->getTailCallKind());
5840
5841 CI->replaceAllUsesWith(NewCall);
5842 CI->eraseFromParent();
5843 return true;
5844}
5845
5846/// Upgrade a call to an old intrinsic. All argument and return casting must be
5847/// provided to seamlessly integrate with existing context.
5848void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5849 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5850 // checks the callee's function type matches. It's likely we need to handle
5851 // type changes here.
5852 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5853 if (!F)
5854 return;
5855
5856 LLVMContext &C = CI->getContext();
5857 IRBuilder<> Builder(CI->getIterator());
5858 if (isa<FPMathOperator>(CI))
5859 Builder.setFastMathFlags(CI->getFastMathFlags());
5860
5861 if (!NewFn) {
5862 // Get the Function's name.
5863 StringRef Name = F->getName();
5864 if (!Name.consume_front("llvm."))
5865 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5866
5867 bool IsX86 = Name.consume_front("x86.");
5868 bool IsNVVM = Name.consume_front("nvvm.");
5869 bool IsAArch64 = Name.consume_front("aarch64.");
5870 bool IsARM = Name.consume_front("arm.");
5871 bool IsAMDGCN = Name.consume_front("amdgcn.");
5872 bool IsDbg = Name.consume_front("dbg.");
5873 bool IsOldSplice =
5874 (Name.consume_front("experimental.vector.splice") ||
5875 Name.consume_front("vector.splice")) &&
5876 !(Name.starts_with(".left") || Name.starts_with(".right"));
5877 Value *Rep = nullptr;
5878
5879 if (!IsX86 && Name == "stackprotectorcheck") {
5880 Rep = nullptr;
5881 } else if (IsNVVM) {
5882 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5883 } else if (IsX86) {
5884 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5885 } else if (IsAArch64) {
5886 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5887 } else if (IsARM) {
5888 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5889 } else if (IsAMDGCN) {
5890 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5891 } else if (IsDbg) {
5892 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5893 } else if (IsOldSplice) {
5894 Rep = upgradeVectorSplice(CI, Builder);
5895 } else if (Name.consume_front("convert.")) {
5896 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5897 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5898 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5899 Rep = nullptr;
5900 } else if (shouldUpgradeVPIntrinsic(Name)) {
5901 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5902 } else {
5903 llvm_unreachable("Unknown function for CallBase upgrade.");
5904 }
5905
5906 if (Rep)
5907 CI->replaceAllUsesWith(Rep);
5908 CI->eraseFromParent();
5909 return;
5910 }
5911
5912 const auto &DefaultCase = [&]() -> void {
5913 if (F == NewFn)
5914 return;
5915
5916 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5917 // Handle generic mangling change.
5918 assert(
5919 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5920 "Unknown function for CallBase upgrade and isn't just a name change");
5921 CI->setCalledFunction(NewFn);
5922 return;
5923 }
5924
5925 // This must be an upgrade from a named to a literal struct.
5926 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5927 assert(OldST != NewFn->getReturnType() &&
5928 "Return type must have changed");
5929 assert(OldST->getNumElements() ==
5930 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5931 "Must have same number of elements");
5932
5933 SmallVector<Value *> Args(CI->args());
5934 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5935 NewCI->setAttributes(CI->getAttributes());
5936 Value *Res = PoisonValue::get(OldST);
5937 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5938 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5939 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5940 }
5941 CI->replaceAllUsesWith(Res);
5942 CI->eraseFromParent();
5943 return;
5944 }
5945
5946 // We're probably about to produce something invalid. Let the verifier catch
5947 // it instead of dying here.
5948 CI->setCalledOperand(
5949 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5950 return;
5951 };
5952 CallInst *NewCall = nullptr;
5953 switch (NewFn->getIntrinsicID()) {
5954 default: {
5955 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5956 return;
5957 DefaultCase();
5958 return;
5959 }
5960 case Intrinsic::arm_neon_vst1:
5961 case Intrinsic::arm_neon_vst2:
5962 case Intrinsic::arm_neon_vst3:
5963 case Intrinsic::arm_neon_vst4:
5964 case Intrinsic::arm_neon_vst2lane:
5965 case Intrinsic::arm_neon_vst3lane:
5966 case Intrinsic::arm_neon_vst4lane: {
5967 SmallVector<Value *, 4> Args(CI->args());
5968 NewCall = Builder.CreateCall(NewFn, Args);
5969 break;
5970 }
5971 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5972 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5973 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5974 LLVMContext &Ctx = F->getParent()->getContext();
5975 SmallVector<Value *, 4> Args(CI->args());
5976 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5977 cast<ConstantInt>(Args[3])->getZExtValue());
5978 NewCall = Builder.CreateCall(NewFn, Args);
5979 break;
5980 }
5981 case Intrinsic::aarch64_sve_ld3_sret:
5982 case Intrinsic::aarch64_sve_ld4_sret:
5983 case Intrinsic::aarch64_sve_ld2_sret: {
5984 // Is this a trivial remangle of the name to support ptr address spaces?
5985 if (isa<StructType>(F->getReturnType())) {
5986 DefaultCase();
5987 return;
5988 }
5989
5990 StringRef Name = F->getName();
5991 Name = Name.substr(5);
5992 unsigned N = StringSwitch<unsigned>(Name)
5993 .StartsWith("aarch64.sve.ld2", 2)
5994 .StartsWith("aarch64.sve.ld3", 3)
5995 .StartsWith("aarch64.sve.ld4", 4)
5996 .Default(0);
5997 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5998 unsigned MinElts = RetTy->getMinNumElements() / N;
5999 SmallVector<Value *, 2> Args(CI->args());
6000 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
6001 Value *Ret = llvm::PoisonValue::get(RetTy);
6002 for (unsigned I = 0; I < N; I++) {
6003 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
6004 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
6005 }
6006 NewCall = dyn_cast<CallInst>(Ret);
6007 break;
6008 }
6009
6010 case Intrinsic::coro_end_async:
6011 case Intrinsic::coro_end: {
6012 SmallVector<Value *, 3> Args(CI->args());
6013 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
6014 Args.push_back(ConstantTokenNone::get(CI->getContext()));
6015 NewCall = Builder.CreateCall(NewFn, Args);
6016
6017 if (!CI->getType()->isVoidTy()) {
6018 if (!CI->use_empty()) {
6019 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
6020 CI->getModule(), Intrinsic::coro_is_in_ramp);
6021 Value *InRamp = Builder.CreateCall(IsInRamp);
6022 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
6023 }
6024 CI->eraseFromParent();
6025 return;
6026 }
6027
6028 break;
6029 }
6030
6031 case Intrinsic::vector_extract: {
6032 StringRef Name = F->getName();
6033 Name = Name.substr(5); // Strip llvm
6034 if (!Name.starts_with("aarch64.sve.tuple.get")) {
6035 DefaultCase();
6036 return;
6037 }
6038 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6039 unsigned MinElts = RetTy->getMinNumElements();
6040 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6041 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
6042 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
6043 break;
6044 }
6045
6046 case Intrinsic::vector_insert: {
6047 StringRef Name = F->getName();
6048 Name = Name.substr(5);
6049 if (!Name.starts_with("aarch64.sve.tuple")) {
6050 DefaultCase();
6051 return;
6052 }
6053 if (Name.starts_with("aarch64.sve.tuple.set")) {
6054 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6055 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
6056 Value *NewIdx =
6057 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
6058 NewCall = Builder.CreateCall(
6059 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
6060 break;
6061 }
6062 if (Name.starts_with("aarch64.sve.tuple.create")) {
6063 unsigned N = StringSwitch<unsigned>(Name)
6064 .StartsWith("aarch64.sve.tuple.create2", 2)
6065 .StartsWith("aarch64.sve.tuple.create3", 3)
6066 .StartsWith("aarch64.sve.tuple.create4", 4)
6067 .Default(0);
6068 assert(N > 1 && "Create is expected to be between 2-4");
6069 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6070 Value *Ret = llvm::PoisonValue::get(RetTy);
6071 unsigned MinElts = RetTy->getMinNumElements() / N;
6072 for (unsigned I = 0; I < N; I++) {
6073 Value *V = CI->getArgOperand(I);
6074 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
6075 }
6076 NewCall = dyn_cast<CallInst>(Ret);
6077 }
6078 break;
6079 }
6080
6081 case Intrinsic::arm_neon_bfdot:
6082 case Intrinsic::arm_neon_bfmmla:
6083 case Intrinsic::arm_neon_bfmlalb:
6084 case Intrinsic::arm_neon_bfmlalt:
6085 case Intrinsic::aarch64_neon_bfdot:
6086 case Intrinsic::aarch64_neon_bfmmla:
6087 case Intrinsic::aarch64_neon_bfmlalb:
6088 case Intrinsic::aarch64_neon_bfmlalt: {
6089 SmallVector<Value *, 3> Args;
6090 assert(CI->arg_size() == 3 &&
6091 "Mismatch between function args and call args");
6092 size_t OperandWidth =
6093 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
6094 assert((OperandWidth == 64 || OperandWidth == 128) &&
6095 "Unexpected operand width");
6096 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6097 auto Iter = CI->args().begin();
6098 Args.push_back(*Iter++);
6099 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6100 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6101 NewCall = Builder.CreateCall(NewFn, Args);
6102 break;
6103 }
6104
6105 case Intrinsic::bitreverse:
6106 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6107 break;
6108
6109 case Intrinsic::ctlz:
6110 case Intrinsic::cttz: {
6111 if (CI->arg_size() != 1) {
6112 DefaultCase();
6113 return;
6114 }
6115
6116 NewCall =
6117 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6118 break;
6119 }
6120
6121 case Intrinsic::objectsize: {
6122 Value *NullIsUnknownSize =
6123 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6124 Value *Dynamic =
6125 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6126 NewCall = Builder.CreateCall(
6127 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6128 break;
6129 }
6130
6131 case Intrinsic::ctpop:
6132 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6133 break;
6134 case Intrinsic::dbg_value: {
6135 StringRef Name = F->getName();
6136 Name = Name.substr(5); // Strip llvm.
6137 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6138 if (Name.starts_with("dbg.addr")) {
6139 DIExpression *Expr = cast<DIExpression>(
6140 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6141 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6142 NewCall =
6143 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6144 MetadataAsValue::get(C, Expr)});
6145 break;
6146 }
6147
6148 // Upgrade from the old version that had an extra offset argument.
6149 assert(CI->arg_size() == 4);
6150 // Drop nonzero offsets instead of attempting to upgrade them.
6151 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6152 if (Offset->isNullValue()) {
6153 NewCall = Builder.CreateCall(
6154 NewFn,
6155 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6156 break;
6157 }
6158 CI->eraseFromParent();
6159 return;
6160 }
6161
6162 case Intrinsic::ptr_annotation:
6163 // Upgrade from versions that lacked the annotation attribute argument.
6164 if (CI->arg_size() != 4) {
6165 DefaultCase();
6166 return;
6167 }
6168
6169 // Create a new call with an added null annotation attribute argument.
6170 NewCall = Builder.CreateCall(
6171 NewFn,
6172 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6173 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6174 NewCall->takeName(CI);
6175 CI->replaceAllUsesWith(NewCall);
6176 CI->eraseFromParent();
6177 return;
6178
6179 case Intrinsic::var_annotation:
6180 // Upgrade from versions that lacked the annotation attribute argument.
6181 if (CI->arg_size() != 4) {
6182 DefaultCase();
6183 return;
6184 }
6185 // Create a new call with an added null annotation attribute argument.
6186 NewCall = Builder.CreateCall(
6187 NewFn,
6188 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6189 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6190 NewCall->takeName(CI);
6191 CI->replaceAllUsesWith(NewCall);
6192 CI->eraseFromParent();
6193 return;
6194
6195 case Intrinsic::riscv_aes32dsi:
6196 case Intrinsic::riscv_aes32dsmi:
6197 case Intrinsic::riscv_aes32esi:
6198 case Intrinsic::riscv_aes32esmi:
6199 case Intrinsic::riscv_sm4ks:
6200 case Intrinsic::riscv_sm4ed: {
6201 // The last argument to these intrinsics used to be i8 and changed to i32.
6202 // The type overload for sm4ks and sm4ed was removed.
6203 Value *Arg2 = CI->getArgOperand(2);
6204 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6205 return;
6206
6207 Value *Arg0 = CI->getArgOperand(0);
6208 Value *Arg1 = CI->getArgOperand(1);
6209 if (CI->getType()->isIntegerTy(64)) {
6210 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6211 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6212 }
6213
6214 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6215 cast<ConstantInt>(Arg2)->getZExtValue());
6216
6217 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6218 Value *Res = NewCall;
6219 if (Res->getType() != CI->getType())
6220 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6221 NewCall->takeName(CI);
6222 CI->replaceAllUsesWith(Res);
6223 CI->eraseFromParent();
6224 return;
6225 }
6226 case Intrinsic::nvvm_mapa_shared_cluster: {
6227 // Create a new call with the correct address space.
6228 NewCall =
6229 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6230 Value *Res = NewCall;
6231 Res = Builder.CreateAddrSpaceCast(
6232 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6233 NewCall->takeName(CI);
6234 CI->replaceAllUsesWith(Res);
6235 CI->eraseFromParent();
6236 return;
6237 }
6238 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6239 SmallVector<Value *, 4> Args(CI->args());
6240 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6241 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6242 Args[0] = Builder.CreateAddrSpaceCast(
6243 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6244
6245 // Append the missing trailing flag_valid_pattern (0 = disabled).
6246 Args.push_back(Builder.getInt32(0));
6247
6248 NewCall = Builder.CreateCall(NewFn, Args);
6249 NewCall->takeName(CI);
6250 CI->replaceAllUsesWith(NewCall);
6251 CI->eraseFromParent();
6252 return;
6253 }
6254 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6255 // (dst, mbar, src, size, ch, flag_ch)
6256 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6257 // i32 0 /* flag_valid_pattern=disabled */)
6258 SmallVector<Value *, 10> Args;
6259 for (unsigned I = 0; I < 4; ++I)
6260 Args.push_back(CI->getArgOperand(I));
6261 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6262 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6263 Args.push_back(CI->getArgOperand(4)); // cache_hint
6264 Args.push_back(CI->getArgOperand(5)); // flag_ch
6265 Args.push_back(Builder.getInt1(false)); // flag_oob
6266 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6267
6268 NewCall = Builder.CreateCall(NewFn, Args);
6269 NewCall->takeName(CI);
6270 CI->replaceAllUsesWith(NewCall);
6271 CI->eraseFromParent();
6272 return;
6273 }
6274 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6275 // Create a new call with the correct address space.
6276 SmallVector<Value *, 4> Args(CI->args());
6277 Args[0] = Builder.CreateAddrSpaceCast(
6278 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6279
6280 NewCall = Builder.CreateCall(NewFn, Args);
6281 NewCall->takeName(CI);
6282 CI->replaceAllUsesWith(NewCall);
6283 CI->eraseFromParent();
6284 return;
6285 }
6286 // clang-format off
6287#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6288 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6289 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6290#undef G2S_CLUSTER_CASE
6291 {
6292 SmallVector<Value *, 16> Args(CI->args());
6293 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6294 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6295 Args[0] = Builder.CreateAddrSpaceCast(
6296 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6297
6298 // Append the missing trailing arguments with default values (cta_group,
6299 // flag_valid_pattern).
6300 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6301 Args.push_back(Builder.getInt32(0));
6302
6303 NewCall = Builder.CreateCall(NewFn, Args);
6304 NewCall->takeName(CI);
6305 CI->replaceAllUsesWith(NewCall);
6306 CI->eraseFromParent();
6307 return;
6308 }
6309
6310#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6311 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6312 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6313#undef G2S_CTA_CASE
6314 {
6315 SmallVector<Value *, 16> Args(CI->args());
6316 // Append the missing trailing flag_valid_pattern argument with default
6317 // value 0.
6318 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6319 "expected only the trailing flag_valid_pattern to be missing");
6320 Args.push_back(Builder.getInt32(0));
6321
6322 NewCall = Builder.CreateCall(NewFn, Args);
6323 NewCall->takeName(CI);
6324 CI->replaceAllUsesWith(NewCall);
6325 CI->eraseFromParent();
6326 return;
6327 }
6328#undef NVVM_TMA_G2S_MODES
6329 // clang-format on
6330
6331 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6332 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6333 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6334 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6335 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6336 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6337 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6338 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6339 StringRef Name = F->getName();
6340 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6341 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6342
6343 SmallVector<Value *, 16> Args(CI->args());
6344 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6345 NewCall = Builder.CreateCall(NewFn, Args);
6346 break;
6347 }
6348 // clang-format off
6349 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cta_space_cluster:
6350 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cluster_space_cluster:
6351 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cta_space_cluster:
6352 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cluster_space_cluster:
6353 case Intrinsic::nvvm_mbarrier_arrive_scope_cta_space_cluster:
6354 case Intrinsic::nvvm_mbarrier_arrive_scope_cluster_space_cluster:
6355 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cta_space_cluster:
6356 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cluster_space_cluster:
6357 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cta_space_cluster:
6358 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cluster_space_cluster:
6359 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cta_space_cluster:
6360 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cluster_space_cluster:
6361 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cta_space_cluster:
6362 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cluster_space_cluster:
6363 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cta_space_cluster:
6364 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cluster_space_cluster:
6365 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cta_space_cluster:
6366 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cluster_space_cluster:
6367 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cta_space_cluster:
6368 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cluster_space_cluster: {
6369 // clang-format on
6370 SmallVector<Value *, 8> Args(CI->args());
6371 Args.push_back(Builder.getInt32(0)); // cta-mask
6372 Args.push_back(Builder.getFalse()); // is-multicast
6373 NewCall = Builder.CreateCall(NewFn, Args);
6374 NewCall->takeName(CI);
6375 CI->replaceAllUsesWith(NewCall);
6376 CI->eraseFromParent();
6377 return;
6378 }
6379 case Intrinsic::nvvm_mbarrier_init: {
6380 SmallVector<Value *, 3> Args(CI->args());
6381 // The .shared variant folded into the overloaded form without gaining an
6382 // operand, so only the pre-layout two-argument form needs one appended.
6383 if (Args.size() == 2)
6384 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6385 NewCall = Builder.CreateCall(NewFn, Args);
6386 break;
6387 }
6388 case Intrinsic::riscv_sha256sig0:
6389 case Intrinsic::riscv_sha256sig1:
6390 case Intrinsic::riscv_sha256sum0:
6391 case Intrinsic::riscv_sha256sum1:
6392 case Intrinsic::riscv_sm3p0:
6393 case Intrinsic::riscv_sm3p1: {
6394 // The last argument to these intrinsics used to be i8 and changed to i32.
6395 // The type overload for sm4ks and sm4ed was removed.
6396 if (!CI->getType()->isIntegerTy(64))
6397 return;
6398
6399 Value *Arg =
6400 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6401
6402 NewCall = Builder.CreateCall(NewFn, Arg);
6403 Value *Res =
6404 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6405 NewCall->takeName(CI);
6406 CI->replaceAllUsesWith(Res);
6407 CI->eraseFromParent();
6408 return;
6409 }
6410
6411 case Intrinsic::x86_xop_vfrcz_ss:
6412 case Intrinsic::x86_xop_vfrcz_sd:
6413 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6414 break;
6415
6416 case Intrinsic::x86_xop_vpermil2pd:
6417 case Intrinsic::x86_xop_vpermil2ps:
6418 case Intrinsic::x86_xop_vpermil2pd_256:
6419 case Intrinsic::x86_xop_vpermil2ps_256: {
6420 SmallVector<Value *, 4> Args(CI->args());
6421 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6422 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6423 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6424 NewCall = Builder.CreateCall(NewFn, Args);
6425 break;
6426 }
6427
6428 case Intrinsic::x86_sse41_ptestc:
6429 case Intrinsic::x86_sse41_ptestz:
6430 case Intrinsic::x86_sse41_ptestnzc: {
6431 // The arguments for these intrinsics used to be v4f32, and changed
6432 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6433 // So, the only thing required is a bitcast for both arguments.
6434 // First, check the arguments have the old type.
6435 Value *Arg0 = CI->getArgOperand(0);
6436 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6437 return;
6438
6439 // Old intrinsic, add bitcasts
6440 Value *Arg1 = CI->getArgOperand(1);
6441
6442 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6443
6444 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6445 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6446
6447 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6448 break;
6449 }
6450
6451 case Intrinsic::x86_rdtscp: {
6452 // This used to take 1 arguments. If we have no arguments, it is already
6453 // upgraded.
6454 if (CI->getNumOperands() == 0)
6455 return;
6456
6457 NewCall = Builder.CreateCall(NewFn);
6458 // Extract the second result and store it.
6459 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6460 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6461 // Replace the original call result with the first result of the new call.
6462 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6463
6464 NewCall->takeName(CI);
6465 CI->replaceAllUsesWith(TSC);
6466 CI->eraseFromParent();
6467 return;
6468 }
6469
6470 case Intrinsic::x86_sse41_insertps:
6471 case Intrinsic::x86_sse41_dppd:
6472 case Intrinsic::x86_sse41_dpps:
6473 case Intrinsic::x86_sse41_mpsadbw:
6474 case Intrinsic::x86_avx_dp_ps_256:
6475 case Intrinsic::x86_avx2_mpsadbw: {
6476 // Need to truncate the last argument from i32 to i8 -- this argument models
6477 // an inherently 8-bit immediate operand to these x86 instructions.
6478 SmallVector<Value *, 4> Args(CI->args());
6479
6480 // Replace the last argument with a trunc.
6481 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6482 NewCall = Builder.CreateCall(NewFn, Args);
6483 break;
6484 }
6485
6486 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6487 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6488 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6489 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6490 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6491 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6492 SmallVector<Value *, 4> Args(CI->args());
6493 unsigned NumElts =
6494 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6495 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6496
6497 NewCall = Builder.CreateCall(NewFn, Args);
6498 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6499
6500 NewCall->takeName(CI);
6501 CI->replaceAllUsesWith(Res);
6502 CI->eraseFromParent();
6503 return;
6504 }
6505
6506 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6507 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6508 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6509 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6510 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6511 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6512 SmallVector<Value *, 4> Args(CI->args());
6513 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6514 if (NewFn->getIntrinsicID() ==
6515 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6516 Args[1] = Builder.CreateBitCast(
6517 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6518
6519 NewCall = Builder.CreateCall(NewFn, Args);
6520 Value *Res = Builder.CreateBitCast(
6521 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6522
6523 NewCall->takeName(CI);
6524 CI->replaceAllUsesWith(Res);
6525 CI->eraseFromParent();
6526 return;
6527 }
6528 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6529 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6530 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6531 SmallVector<Value *, 4> Args(CI->args());
6532 unsigned NumElts =
6533 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6534 Args[1] = Builder.CreateBitCast(
6535 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6536 Args[2] = Builder.CreateBitCast(
6537 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6538
6539 NewCall = Builder.CreateCall(NewFn, Args);
6540 break;
6541 }
6542
6543 case Intrinsic::amdgcn_asyncmark:
6544 case Intrinsic::amdgcn_wait_asyncmark: {
6545 // Legacy asyncmark intrinsics missed the stage mask operand. Append a zero
6546 // mask, which names every stage and so preserves the behavior they had.
6547 SmallVector<Value *, 2> Args(CI->args());
6548 Args.push_back(Builder.getInt32(0));
6549
6550 SmallVector<OperandBundleDef, 1> Bundles;
6551 CI->getOperandBundlesAsDefs(Bundles);
6552
6553 NewCall = Builder.CreateCall(NewFn, Args, Bundles);
6554 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6555 NewCall->setCallingConv(CI->getCallingConv());
6556 NewCall->setAttributes(CI->getAttributes());
6557 NewCall->copyMetadata(*CI);
6558 break;
6559 }
6560
6561 case Intrinsic::thread_pointer: {
6562 NewCall = Builder.CreateCall(NewFn, {});
6563 break;
6564 }
6565
6566 case Intrinsic::memcpy:
6567 case Intrinsic::memmove:
6568 case Intrinsic::memset: {
6569 // We have to make sure that the call signature is what we're expecting.
6570 // We only want to change the old signatures by removing the alignment arg:
6571 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6572 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6573 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6574 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6575 // Note: i8*'s in the above can be any pointer type
6576 if (CI->arg_size() != 5) {
6577 DefaultCase();
6578 return;
6579 }
6580 // Remove alignment argument (3), and add alignment attributes to the
6581 // dest/src pointers.
6582 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6583 CI->getArgOperand(2), CI->getArgOperand(4)};
6584 NewCall = Builder.CreateCall(NewFn, Args);
6585 AttributeList OldAttrs = CI->getAttributes();
6586 AttributeList NewAttrs = AttributeList::get(
6587 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6588 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6589 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6590 NewCall->setAttributes(NewAttrs);
6591 auto *MemCI = cast<MemIntrinsic>(NewCall);
6592 // All mem intrinsics support dest alignment.
6593 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6594 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6595 // Memcpy/Memmove also support source alignment.
6596 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6597 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6598 break;
6599 }
6600
6601 case Intrinsic::masked_load:
6602 case Intrinsic::masked_gather:
6603 case Intrinsic::masked_store:
6604 case Intrinsic::masked_scatter: {
6605 if (CI->arg_size() != 4) {
6606 DefaultCase();
6607 return;
6608 }
6609
6610 auto GetMaybeAlign = [](Value *Op) {
6611 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6612 uint64_t Val = CI->getZExtValue();
6613 if (Val == 0)
6614 return MaybeAlign();
6615 if (isPowerOf2_64(Val))
6616 return MaybeAlign(Val);
6617 }
6618 reportFatalUsageError("Invalid alignment argument");
6619 };
6620 auto GetAlign = [&](Value *Op) {
6621 MaybeAlign Align = GetMaybeAlign(Op);
6622 if (Align)
6623 return *Align;
6624 reportFatalUsageError("Invalid zero alignment argument");
6625 };
6626
6627 const DataLayout &DL = CI->getDataLayout();
6628 switch (NewFn->getIntrinsicID()) {
6629 case Intrinsic::masked_load:
6630 NewCall = Builder.CreateMaskedLoad(
6631 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6632 CI->getArgOperand(2), CI->getArgOperand(3));
6633 break;
6634 case Intrinsic::masked_gather:
6635 NewCall = Builder.CreateMaskedGather(
6636 CI->getType(), CI->getArgOperand(0),
6637 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6638 CI->getType()->getScalarType()),
6639 CI->getArgOperand(2), CI->getArgOperand(3));
6640 break;
6641 case Intrinsic::masked_store:
6642 NewCall = Builder.CreateMaskedStore(
6643 CI->getArgOperand(0), CI->getArgOperand(1),
6644 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6645 break;
6646 case Intrinsic::masked_scatter:
6647 NewCall = Builder.CreateMaskedScatter(
6648 CI->getArgOperand(0), CI->getArgOperand(1),
6649 DL.getValueOrABITypeAlignment(
6650 GetMaybeAlign(CI->getArgOperand(2)),
6651 CI->getArgOperand(0)->getType()->getScalarType()),
6652 CI->getArgOperand(3));
6653 break;
6654 default:
6655 llvm_unreachable("Unexpected intrinsic ID");
6656 }
6657 // Previous metadata is still valid.
6658 NewCall->copyMetadata(*CI);
6659 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6660 break;
6661 }
6662
6663 case Intrinsic::lifetime_start:
6664 case Intrinsic::lifetime_end: {
6665 if (CI->arg_size() != 2) {
6666 DefaultCase();
6667 return;
6668 }
6669
6670 Value *Ptr = CI->getArgOperand(1);
6671 // Try to strip pointer casts, such that the lifetime works on an alloca.
6672 Ptr = Ptr->stripPointerCasts();
6673 if (isa<AllocaInst>(Ptr)) {
6674 // Don't use NewFn, as we might have looked through an addrspacecast.
6675 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6676 NewCall = Builder.CreateLifetimeStart(Ptr);
6677 else
6678 NewCall = Builder.CreateLifetimeEnd(Ptr);
6679 break;
6680 }
6681
6682 // Otherwise remove the lifetime marker.
6683 CI->eraseFromParent();
6684 return;
6685 }
6686
6687 case Intrinsic::x86_avx512_vpdpbusd_128:
6688 case Intrinsic::x86_avx512_vpdpbusd_256:
6689 case Intrinsic::x86_avx512_vpdpbusd_512:
6690 case Intrinsic::x86_avx512_vpdpbusds_128:
6691 case Intrinsic::x86_avx512_vpdpbusds_256:
6692 case Intrinsic::x86_avx512_vpdpbusds_512:
6693 case Intrinsic::x86_avx2_vpdpbssd_128:
6694 case Intrinsic::x86_avx2_vpdpbssd_256:
6695 case Intrinsic::x86_avx10_vpdpbssd_512:
6696 case Intrinsic::x86_avx2_vpdpbssds_128:
6697 case Intrinsic::x86_avx2_vpdpbssds_256:
6698 case Intrinsic::x86_avx10_vpdpbssds_512:
6699 case Intrinsic::x86_avx2_vpdpbsud_128:
6700 case Intrinsic::x86_avx2_vpdpbsud_256:
6701 case Intrinsic::x86_avx10_vpdpbsud_512:
6702 case Intrinsic::x86_avx2_vpdpbsuds_128:
6703 case Intrinsic::x86_avx2_vpdpbsuds_256:
6704 case Intrinsic::x86_avx10_vpdpbsuds_512:
6705 case Intrinsic::x86_avx2_vpdpbuud_128:
6706 case Intrinsic::x86_avx2_vpdpbuud_256:
6707 case Intrinsic::x86_avx10_vpdpbuud_512:
6708 case Intrinsic::x86_avx2_vpdpbuuds_128:
6709 case Intrinsic::x86_avx2_vpdpbuuds_256:
6710 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6711 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6712 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6713 CI->getArgOperand(2)};
6714 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6715 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6716 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6717
6718 NewCall = Builder.CreateCall(NewFn, Args);
6719 break;
6720 }
6721 case Intrinsic::x86_avx512_vpdpwssd_128:
6722 case Intrinsic::x86_avx512_vpdpwssd_256:
6723 case Intrinsic::x86_avx512_vpdpwssd_512:
6724 case Intrinsic::x86_avx512_vpdpwssds_128:
6725 case Intrinsic::x86_avx512_vpdpwssds_256:
6726 case Intrinsic::x86_avx512_vpdpwssds_512:
6727 case Intrinsic::x86_avx2_vpdpwsud_128:
6728 case Intrinsic::x86_avx2_vpdpwsud_256:
6729 case Intrinsic::x86_avx10_vpdpwsud_512:
6730 case Intrinsic::x86_avx2_vpdpwsuds_128:
6731 case Intrinsic::x86_avx2_vpdpwsuds_256:
6732 case Intrinsic::x86_avx10_vpdpwsuds_512:
6733 case Intrinsic::x86_avx2_vpdpwusd_128:
6734 case Intrinsic::x86_avx2_vpdpwusd_256:
6735 case Intrinsic::x86_avx10_vpdpwusd_512:
6736 case Intrinsic::x86_avx2_vpdpwusds_128:
6737 case Intrinsic::x86_avx2_vpdpwusds_256:
6738 case Intrinsic::x86_avx10_vpdpwusds_512:
6739 case Intrinsic::x86_avx2_vpdpwuud_128:
6740 case Intrinsic::x86_avx2_vpdpwuud_256:
6741 case Intrinsic::x86_avx10_vpdpwuud_512:
6742 case Intrinsic::x86_avx2_vpdpwuuds_128:
6743 case Intrinsic::x86_avx2_vpdpwuuds_256:
6744 case Intrinsic::x86_avx10_vpdpwuuds_512:
6745 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6746 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6747 CI->getArgOperand(2)};
6748 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6749 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6750 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6751
6752 NewCall = Builder.CreateCall(NewFn, Args);
6753 break;
6754 }
6755 assert(NewCall && "Should have either set this variable or returned through "
6756 "the default case");
6757 NewCall->takeName(CI);
6758 CI->replaceAllUsesWith(NewCall);
6759 CI->eraseFromParent();
6760}
6761
6762void llvm::UpgradeCallsToIntrinsic(Function *F) {
6763 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6764
6765 // Check if this function should be upgraded and get the replacement function
6766 // if there is one.
6767 Function *NewFn;
6768 if (UpgradeIntrinsicFunction(F, NewFn)) {
6769 // Replace all users of the old function with the new function or new
6770 // instructions. This is not a range loop because the call is deleted.
6771 for (User *U : make_early_inc_range(F->users()))
6772 if (CallBase *CB = dyn_cast<CallBase>(U))
6773 UpgradeIntrinsicCall(CB, NewFn);
6774
6775 // Remove old function, no longer used, from the module.
6776 if (F != NewFn)
6777 F->eraseFromParent();
6778 }
6779}
6780
6781MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6782 const unsigned NumOperands = MD.getNumOperands();
6783 if (NumOperands == 0)
6784 return &MD; // Invalid, punt to a verifier error.
6785
6786 // Check if the tag uses struct-path aware TBAA format.
6787 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6788 return &MD;
6789
6790 auto &Context = MD.getContext();
6791 if (NumOperands == 3) {
6792 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6793 MDNode *ScalarType = MDNode::get(Context, Elts);
6794 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6795 Metadata *Elts2[] = {ScalarType, ScalarType,
6796 ConstantAsMetadata::get(
6797 Constant::getNullValue(Type::getInt64Ty(Context))),
6798 MD.getOperand(2)};
6799 return MDNode::get(Context, Elts2);
6800 }
6801 // Create a MDNode <MD, MD, offset 0>
6802 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6803 Type::getInt64Ty(Context)))};
6804 return MDNode::get(Context, Elts);
6805}
6806
6807MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6808 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6809 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6810 unsigned NumOperands = MD.getNumOperands();
6811 if (NumOperands == 0 || NumOperands % 3 != 0)
6812 return &MD; // Malformed; leave it for the verifier to reject.
6813
6814 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6815 bool Changed = false;
6816 for (unsigned I = 2; I < NumOperands; I += 3) {
6817 auto *Tag = dyn_cast_or_null<MDNode>(Elts[I]);
6818 if (!Tag)
6819 continue;
6820 MDNode *Upgraded = UpgradeTBAANode(*Tag);
6821 if (Upgraded == Tag)
6822 continue;
6823 Elts[I] = Upgraded;
6824 Changed = true;
6825 }
6826 return Changed ? MDNode::get(MD.getContext(), Elts) : &MD;
6827}
6828
6829Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6830 Instruction *&Temp) {
6831 if (Opc != Instruction::BitCast)
6832 return nullptr;
6833
6834 Temp = nullptr;
6835 Type *SrcTy = V->getType();
6836 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6837 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6838 LLVMContext &Context = V->getContext();
6839
6840 // We have no information about target data layout, so we assume that
6841 // the maximum pointer size is 64bit.
6842 Type *MidTy = Type::getInt64Ty(Context);
6843 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6844
6845 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6846 }
6847
6848 return nullptr;
6849}
6850
6851Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6852 if (Opc != Instruction::BitCast)
6853 return nullptr;
6854
6855 Type *SrcTy = C->getType();
6856 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6857 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6858 LLVMContext &Context = C->getContext();
6859
6860 // We have no information about target data layout, so we assume that
6861 // the maximum pointer size is 64bit.
6862 Type *MidTy = Type::getInt64Ty(Context);
6863
6864 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6865 DestTy);
6866 }
6867
6868 return nullptr;
6869}
6870
6871static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6872 if (Flag.getNumOperands() < 3)
6873 return std::nullopt;
6874 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6875 return Name->getString();
6876 return std::nullopt;
6877}
6878
6879/// Check the debug info version number, if it is out-dated, drop the debug
6880/// info. Return true if module is modified.
6881bool llvm::UpgradeDebugInfo(Module &M) {
6882 if (!getIROptions(M.getContext()).auto_upgrade_debug_info)
6883 return false;
6884
6885 llvm::TimeTraceScope timeScope("Upgrade debug info");
6886 // We need to get metadata before the module is verified (i.e., getModuleFlag
6887 // makes assumptions that we haven't verified yet). Carefully extract the flag
6888 // from the metadata.
6889 unsigned Version = 0;
6890 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6891 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6892 if (auto Name = getModuleFlagNameSafely(*Flag))
6893 return *Name == "Debug Info Version";
6894 return false;
6895 });
6896 if (OpIt != ModFlags->op_end()) {
6897 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6898 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6899 Version = CI->getZExtValue();
6900 }
6901 }
6902
6903 if (Version == DEBUG_METADATA_VERSION) {
6904 bool BrokenDebugInfo = false;
6905 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6906 report_fatal_error("Broken module found, compilation aborted!");
6907 if (!BrokenDebugInfo)
6908 // Everything is ok.
6909 return false;
6910 else {
6911 // Diagnose malformed debug info.
6912 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6913 M.getContext().diagnose(Diag);
6914 }
6915 }
6916 bool Modified = StripDebugInfo(M);
6917 if (Modified && Version != DEBUG_METADATA_VERSION) {
6918 // Diagnose a version mismatch.
6919 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6920 M.getContext().diagnose(DiagVersion);
6921 }
6922 return Modified;
6923}
6924
6925static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6926 GlobalValue *GV, const Metadata *V) {
6927 Function *F = cast<Function>(GV);
6928
6929 constexpr StringLiteral DefaultValue = "1";
6930 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6931 unsigned Length = 0;
6932
6933 if (F->hasFnAttribute(Attr)) {
6934 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6935 // parse these elements placing them into Vect3
6936 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6937 for (; Length < 3 && !S.empty(); Length++) {
6938 auto [Part, Rest] = S.split(',');
6939 Vect3[Length] = Part.trim();
6940 S = Rest;
6941 }
6942 }
6943
6944 const unsigned Dim = DimC - 'x';
6945 assert(Dim < 3 && "Unexpected dim char");
6946
6947 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6948
6949 // local variable required for StringRef in Vect3 to point to.
6950 const std::string VStr = llvm::utostr(VInt);
6951 Vect3[Dim] = VStr;
6952 Length = std::max(Length, Dim + 1);
6953
6954 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6955 F->addFnAttr(Attr, NewAttr);
6956}
6957
6958static inline bool isXYZ(StringRef S) {
6959 return S == "x" || S == "y" || S == "z";
6960}
6961
6962bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6963 const Metadata *V) {
6964 if (K == "kernel") {
6965 if (!mdconst::extract<ConstantInt>(V)->isZero())
6966 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6967 return true;
6968 }
6969 if (K == "align") {
6970 // V is a bitfeild specifying two 16-bit values. The alignment value is
6971 // specfied in low 16-bits, The index is specified in the high bits. For the
6972 // index, 0 indicates the return value while higher values correspond to
6973 // each parameter (idx = param + 1).
6974 const uint64_t AlignIdxValuePair =
6975 mdconst::extract<ConstantInt>(V)->getZExtValue();
6976 const unsigned Idx = (AlignIdxValuePair >> 16);
6977 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6978 cast<Function>(GV)->addAttributeAtIndex(
6979 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6980 return true;
6981 }
6982 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6983 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6984 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6985 return true;
6986 }
6987 if (K == "minctasm") {
6988 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6989 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6990 return true;
6991 }
6992 if (K == "maxnreg") {
6993 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6994 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6995 return true;
6996 }
6997 if (K.consume_front("maxntid") && isXYZ(K)) {
6998 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6999 return true;
7000 }
7001 if (K.consume_front("reqntid") && isXYZ(K)) {
7002 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
7003 return true;
7004 }
7005 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
7006 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
7007 return true;
7008 }
7009 if (K == "grid_constant") {
7010 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
7011 for (const auto &Op : cast<MDNode>(V)->operands()) {
7012 // For some reason, the index is 1-based in the metadata. Good thing we're
7013 // able to auto-upgrade it!
7014 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
7015 cast<Function>(GV)->addParamAttr(Index, Attr);
7016 }
7017 return true;
7018 }
7019
7020 return false;
7021}
7022
7023void llvm::UpgradeNVVMAnnotations(Module &M) {
7024 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
7025 if (!NamedMD)
7026 return;
7027
7028 SmallVector<MDNode *, 8> NewNodes;
7029 SmallPtrSet<const MDNode *, 8> SeenNodes;
7030 for (MDNode *MD : NamedMD->operands()) {
7031 if (!SeenNodes.insert(MD).second)
7032 continue;
7033
7034 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
7035 if (!GV)
7036 continue;
7037
7038 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
7039
7040 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
7041 // Each nvvm.annotations metadata entry will be of the following form:
7042 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
7043 // start index = 1, to skip the global variable key
7044 // increment = 2, to skip the value for each property-value pairs
7045 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
7046 MDString *K = cast<MDString>(MD->getOperand(j));
7047 const MDOperand &V = MD->getOperand(j + 1);
7048 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
7049 if (!Upgraded)
7050 NewOperands.append({K, V});
7051 }
7052
7053 if (NewOperands.size() > 1)
7054 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
7055 }
7056
7057 NamedMD->clearOperands();
7058 for (MDNode *N : NewNodes)
7059 NamedMD->addOperand(N);
7060}
7061
7062/// This checks for objc retain release marker which should be upgraded. It
7063/// returns true if module is modified.
7064static bool upgradeRetainReleaseMarker(Module &M) {
7065 bool Changed = false;
7066 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
7067 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
7068 if (ModRetainReleaseMarker) {
7069 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
7070 if (Op) {
7071 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
7072 if (ID) {
7073 SmallVector<StringRef, 4> ValueComp;
7074 ID->getString().split(ValueComp, "#");
7075 if (ValueComp.size() == 2) {
7076 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
7077 ID = MDString::get(M.getContext(), NewValue);
7078 }
7079 M.addModuleFlag(Module::Error, MarkerKey, ID);
7080 M.eraseNamedMetadata(ModRetainReleaseMarker);
7081 Changed = true;
7082 }
7083 }
7084 }
7085 return Changed;
7086}
7087
7088void llvm::UpgradeARCRuntime(Module &M) {
7089 // This lambda converts normal function calls to ARC runtime functions to
7090 // intrinsic calls.
7091 auto UpgradeToIntrinsic = [&](const char *OldFunc,
7092 llvm::Intrinsic::ID IntrinsicFunc) {
7093 Function *Fn = M.getFunction(OldFunc);
7094
7095 if (!Fn)
7096 return;
7097
7098 Function *NewFn =
7099 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
7100
7101 for (User *U : make_early_inc_range(Fn->users())) {
7102 CallInst *CI = dyn_cast<CallInst>(U);
7103 if (!CI || CI->getCalledFunction() != Fn)
7104 continue;
7105
7106 IRBuilder<> Builder(CI->getIterator());
7107 FunctionType *NewFuncTy = NewFn->getFunctionType();
7108 SmallVector<Value *, 2> Args;
7109
7110 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7111 // value to the return type of the old function.
7112 if (NewFuncTy->getReturnType() != CI->getType() &&
7113 !CastInst::castIsValid(Instruction::BitCast, CI,
7114 NewFuncTy->getReturnType()))
7115 continue;
7116
7117 bool InvalidCast = false;
7118
7119 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7120 Value *Arg = CI->getArgOperand(I);
7121
7122 // Bitcast argument to the parameter type of the new function if it's
7123 // not a variadic argument.
7124 if (I < NewFuncTy->getNumParams()) {
7125 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7126 // to the parameter type of the new function.
7127 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
7128 NewFuncTy->getParamType(I))) {
7129 InvalidCast = true;
7130 break;
7131 }
7132 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
7133 }
7134 Args.push_back(Arg);
7135 }
7136
7137 if (InvalidCast)
7138 continue;
7139
7140 // Create a call instruction that calls the new function.
7141 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7142 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
7143 NewCall->takeName(CI);
7144
7145 // Bitcast the return value back to the type of the old call.
7146 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
7147
7148 if (!CI->use_empty())
7149 CI->replaceAllUsesWith(NewRetVal);
7150 CI->eraseFromParent();
7151 }
7152
7153 if (Fn->use_empty())
7154 Fn->eraseFromParent();
7155 };
7156
7157 // Unconditionally convert a call to "clang.arc.use" to a call to
7158 // "llvm.objc.clang.arc.use".
7159 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7160
7161 // Upgrade the retain release marker. If there is no need to upgrade
7162 // the marker, that means either the module is already new enough to contain
7163 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7164 if (!upgradeRetainReleaseMarker(M))
7165 return;
7166
7167 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7168 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7169 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7170 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7171 {"objc_autoreleaseReturnValue",
7172 llvm::Intrinsic::objc_autoreleaseReturnValue},
7173 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7174 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7175 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7176 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7177 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7178 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7179 {"objc_release", llvm::Intrinsic::objc_release},
7180 {"objc_retain", llvm::Intrinsic::objc_retain},
7181 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7182 {"objc_retainAutoreleaseReturnValue",
7183 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7184 {"objc_retainAutoreleasedReturnValue",
7185 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7186 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7187 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7188 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7189 {"objc_unsafeClaimAutoreleasedReturnValue",
7190 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7191 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7192 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7193 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7194 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7195 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7196 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7197 {"objc_arc_annotation_topdown_bbstart",
7198 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7199 {"objc_arc_annotation_topdown_bbend",
7200 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7201 {"objc_arc_annotation_bottomup_bbstart",
7202 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7203 {"objc_arc_annotation_bottomup_bbend",
7204 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7205
7206 for (auto &I : RuntimeFuncs)
7207 UpgradeToIntrinsic(I.first, I.second);
7208}
7209
7210// Upgrade the way signing of pointers to init/fini functions is described.
7211//
7212// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7213// constants, if signing was requested. After the upgrade, these arrays contain
7214// plain function pointers and the desired signing schema is described via a
7215// pair of module flags.
7216//
7217// Note that the upgrade is only performed if all elements of *both* arrays
7218// agree on a common signing schema.
7219static bool upgradePtrauthInitFiniArrays(Module &M) {
7220 // As we cannot always decide whether the particular module should have
7221 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7222 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7223 // returns as soon as it spots any non-signed init/fini pointer: either we
7224 // should request non-signed pointers (safe to omit both flags) or there is
7225 // no common schema (and thus we do not modify anything).
7226 //
7227 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7228 // or whether we should request address diversity in addition to the basic
7229 // constant diversity. There is no value representing "decided not to sign"
7230 // for the reasons explained above.
7231 std::optional<bool> UseAddressDisc;
7232
7233 // Do not attempt upgrading if the new module flags already exist.
7234 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7235 for (const MDNode *Flag : ModFlags->operands()) {
7236 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7237 if (Name && (*Name == "ptrauth-init-fini" ||
7238 *Name == "ptrauth-init-fini-address-discrimination"))
7239 return false;
7240 }
7241 }
7242
7243 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7244 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7245 constexpr unsigned ExpectedAddressMarker = 1;
7246
7247 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7248 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7249 return nullptr; // Nothing to upgrade or unknown pattern found.
7250
7251 bool HasAddressDisc;
7252 if (!CPA->hasAddressDiscriminator())
7253 HasAddressDisc = false;
7254 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7255 HasAddressDisc = true;
7256 else
7257 return nullptr; // Unknown pattern.
7258
7259 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7260 return nullptr; // Disagreement with the decided mode.
7261
7262 UseAddressDisc = HasAddressDisc;
7263 return CPA->getPointer();
7264 };
7265
7266 // Do not apply any changes until we know the upgrade is non-ambiguous.
7267 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7268 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7269
7270 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7271 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7272 if (!GV || !GV->hasInitializer())
7273 continue; // Skip, but it is okay to upgrade the other variable.
7274
7275 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7276 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7277 return false;
7278
7279 std::vector<Constant *> NewStructors;
7280 NewStructors.reserve(OldStructorsArray->getNumOperands());
7281
7282 for (Use &U : OldStructorsArray->operands()) {
7283 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7284 if (!Structor || Structor->getNumOperands() != 3)
7285 return false;
7286
7287 Constant *Prio = Structor->getOperand(0);
7288 Constant *Func = Structor->getOperand(1);
7289 Constant *Arg = Structor->getOperand(2);
7290
7291 Func = UpgradeSinglePointer(Func);
7292 if (!Func)
7293 return false;
7294
7295 NewStructors.push_back(
7296 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7297 }
7298
7299 Constant *NewInit =
7300 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7301 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7302 }
7303
7304 if (GlobalArraysToUpgrade.empty())
7305 return false;
7306 assert(UseAddressDisc.has_value());
7307
7308 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7309 GV->setInitializer(NewInit);
7310
7311 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7312 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7313 *UseAddressDisc);
7314
7315 return true;
7316}
7317
7318bool llvm::UpgradeModuleFlags(Module &M) {
7319 bool Changed = false;
7320 Changed |= upgradePtrauthInitFiniArrays(M);
7321
7322 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7323 if (!ModFlags)
7324 return Changed;
7325
7326 bool HasObjCFlag = false, HasClassProperties = false;
7327 bool HasSwiftVersionFlag = false;
7328 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7329 uint32_t SwiftABIVersion;
7330 auto Int8Ty = Type::getInt8Ty(M.getContext());
7331 auto Int32Ty = Type::getInt32Ty(M.getContext());
7332
7333 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7334 MDNode *Op = ModFlags->getOperand(I);
7335 if (Op->getNumOperands() != 3)
7336 continue;
7337 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7338 if (!ID)
7339 continue;
7340 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7341 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7342 Type::getInt32Ty(M.getContext()), B)),
7343 MDString::get(M.getContext(), ID->getString()),
7344 Op->getOperand(2)};
7345 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7346 Changed = true;
7347 };
7348
7349 if (ID->getString() == "Objective-C Image Info Version")
7350 HasObjCFlag = true;
7351 if (ID->getString() == "Objective-C Class Properties")
7352 HasClassProperties = true;
7353 // Upgrade PIC from Error/Max to Min.
7354 if (ID->getString() == "PIC Level") {
7355 if (auto *Behavior =
7356 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7357 uint64_t V = Behavior->getLimitedValue();
7358 if (V == Module::Error || V == Module::Max)
7359 SetBehavior(Module::Min);
7360 }
7361 }
7362 // Upgrade "PIE Level" from Error to Max.
7363 if (ID->getString() == "PIE Level")
7364 if (auto *Behavior =
7365 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7366 if (Behavior->getLimitedValue() == Module::Error)
7367 SetBehavior(Module::Max);
7368
7369 // Upgrade branch protection and return address signing module flags. The
7370 // module flag behavior for these fields were Error and now they are Min.
7371 // The one exception is "sign-return-address-harden".
7372 if (ID->getString() == "branch-target-enforcement" ||
7373 (ID->getString().starts_with("sign-return-address") &&
7374 ID->getString() != "sign-return-address-harden")) {
7375 if (auto *Behavior =
7376 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7377 if (Behavior->getLimitedValue() == Module::Error) {
7378 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7379 Metadata *Ops[3] = {
7380 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7381 Op->getOperand(1), Op->getOperand(2)};
7382 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7383 Changed = true;
7384 }
7385 }
7386 }
7387
7388 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7389 // section name so that llvm-lto will not complain about mismatching
7390 // module flags that is functionally the same.
7391 if (ID->getString() == "Objective-C Image Info Section") {
7392 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7393 SmallVector<StringRef, 4> ValueComp;
7394 Value->getString().split(ValueComp, " ");
7395 if (ValueComp.size() != 1) {
7396 std::string NewValue;
7397 for (auto &S : ValueComp)
7398 NewValue += S.str();
7399 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7400 MDString::get(M.getContext(), NewValue)};
7401 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7402 Changed = true;
7403 }
7404 }
7405 }
7406
7407 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7408 // If the higher bits are set, it adds new module flag for swift info.
7409 if (ID->getString() == "Objective-C Garbage Collection") {
7410 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7411 if (Md) {
7412 assert(Md->getValue() && "Expected non-empty metadata");
7413 auto Type = Md->getValue()->getType();
7414 if (Type == Int8Ty)
7415 continue;
7416 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7417 if ((Val & 0xff) != Val) {
7418 HasSwiftVersionFlag = true;
7419 SwiftABIVersion = (Val & 0xff00) >> 8;
7420 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7421 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7422 }
7423 Metadata *Ops[3] = {
7424 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7425 Op->getOperand(1),
7426 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7427 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7428 Changed = true;
7429 }
7430 }
7431
7432 if (ID->getString() == "amdgpu_code_object_version") {
7433 Metadata *Ops[3] = {
7434 Op->getOperand(0),
7435 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7436 Op->getOperand(2)};
7437 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7438 Changed = true;
7439 }
7440
7441 // clang/PowerPC used to use "float-abi" to describe the long double format;
7442 // it has been renamed to "long-double-type", with its values changed to the
7443 // corresponding IR floating-point type names.
7444 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7445 StringRef Format;
7446 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7447 Format = S->getString();
7448
7449 // The "float-abi" key is now reserved for the target-independent
7450 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7451 // (including unrecognized ones, which were never valid) to the default.
7452 if (!FloatABI::parseABIType(Format)) {
7453 LongDoubleFormat NewFormat =
7454 StringSwitch<LongDoubleFormat>(Format)
7455 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7456 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7457 .Default(LongDoubleFormat::PPCDoubleDouble);
7458 Metadata *Ops[3] = {
7459 Op->getOperand(0),
7460 MDString::get(M.getContext(), "long-double-type"),
7461 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7462 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7463 Changed = true;
7464 }
7465 }
7466 }
7467
7468 // "Objective-C Class Properties" is recently added for Objective-C. We
7469 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7470 // flag of value 0, so we can correclty downgrade this flag when trying to
7471 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7472 // this module flag.
7473 if (HasObjCFlag && !HasClassProperties) {
7474 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7475 (uint32_t)0);
7476 Changed = true;
7477 }
7478
7479 if (HasSwiftVersionFlag) {
7480 M.addModuleFlag(Module::Error, "Swift ABI Version",
7481 SwiftABIVersion);
7482 M.addModuleFlag(Module::Error, "Swift Major Version",
7483 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7484 M.addModuleFlag(Module::Error, "Swift Minor Version",
7485 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7486 Changed = true;
7487 }
7488
7489 return Changed;
7490}
7491
7492bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7493 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7494 // If this metadata has operands, we expect all of them to be either from
7495 // before or from after the format change handled here, so we can bail out
7496 // fast if the first (if any) operands is of the new format.
7497 auto MatchesVersion = [](const MDNode *Op) {
7498 return Op->getNumOperands() >= 3 &&
7499 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7500 cast<ConstantAsMetadata>(Op->getOperand(2))
7501 ->getType()
7502 ->isIntegerTy(64);
7503 };
7504
7505 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7506 MatchesVersion(CFIConsts->getOperand(0)))
7507 return false;
7508
7509 bool Changed = false;
7510 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7511 MDNode *Op = CFIConsts->getOperand(I);
7512 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7513 assert(Op->getNumOperands() >= 2 &&
7514 "Expected at least 2 operands - name and linkage type");
7515 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7516 StringRef Name = NameMD->getString();
7517 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7518 GlobalValue::dropLLVMManglingEscape(Name));
7519
7520 SmallVector<Metadata *, 4> Elts;
7521 Elts.push_back(Op->getOperand(0));
7522 Elts.push_back(Op->getOperand(1));
7523 Elts.push_back(ConstantAsMetadata::get(
7524 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7525
7526 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7527 Elts.push_back(Op->getOperand(J));
7528
7529 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7530 Changed = true;
7531 }
7532
7533 return Changed;
7534}
7535
7536void llvm::UpgradeSectionAttributes(Module &M) {
7537 auto TrimSpaces = [](StringRef Section) -> std::string {
7538 SmallVector<StringRef, 5> Components;
7539 Section.split(Components, ',');
7540
7541 SmallString<32> Buffer;
7542 raw_svector_ostream OS(Buffer);
7543
7544 for (auto Component : Components)
7545 OS << ',' << Component.trim();
7546
7547 return std::string(OS.str().substr(1));
7548 };
7549
7550 for (auto &GV : M.globals()) {
7551 if (!GV.hasSection())
7552 continue;
7553
7554 StringRef Section = GV.getSection();
7555
7556 if (!Section.starts_with("__DATA, __objc_catlist"))
7557 continue;
7558
7559 // __DATA, __objc_catlist, regular, no_dead_strip
7560 // __DATA,__objc_catlist,regular,no_dead_strip
7561 GV.setSection(TrimSpaces(Section));
7562 }
7563}
7564
7565namespace {
7566// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7567// callsites within a function that did not also have the strictfp attribute.
7568// Since 10.0, if strict FP semantics are needed within a function, the
7569// function must have the strictfp attribute and all calls within the function
7570// must also have the strictfp attribute. This latter restriction is
7571// necessary to prevent unwanted libcall simplification when a function is
7572// being cloned (such as for inlining).
7573//
7574// The "dangling" strictfp attribute usage was only used to prevent constant
7575// folding and other libcall simplification. The nobuiltin attribute on the
7576// callsite has the same effect.
7577struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7578 StrictFPUpgradeVisitor() = default;
7579
7580 void visitCallBase(CallBase &Call) {
7581 if (!Call.isStrictFP())
7582 return;
7583 if (isa<ConstrainedFPIntrinsic>(&Call))
7584 return;
7585 // If we get here, the caller doesn't have the strictfp attribute
7586 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7587 Call.removeFnAttr(Attribute::StrictFP);
7588 Call.addFnAttr(Attribute::NoBuiltin);
7589 }
7590};
7591
7592/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7593struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7594 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7595 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7596
7597 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7598 if (!RMW.isFloatingPointOperation())
7599 return;
7600
7601 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7602 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7603 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7604 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7605 }
7606};
7607} // namespace
7608
7609void llvm::UpgradeFunctionAttributes(Function &F) {
7610 // If a function definition doesn't have the strictfp attribute,
7611 // convert any callsite strictfp attributes to nobuiltin.
7612 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7613 StrictFPUpgradeVisitor SFPV;
7614 SFPV.visit(F);
7615 }
7616
7617 // Remove all incompatibile attributes from function.
7618 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7619 F.getReturnType(), F.getAttributes().getRetAttrs()));
7620 for (auto &Arg : F.args())
7621 Arg.removeAttrs(
7622 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7623
7624 bool AddingAttrs = false, RemovingAttrs = false;
7625 AttrBuilder AttrsToAdd(F.getContext());
7626 AttributeMask AttrsToRemove;
7627
7628 // Older versions of LLVM treated an "implicit-section-name" attribute
7629 // similarly to directly setting the section on a Function.
7630 if (Attribute A = F.getFnAttribute("implicit-section-name");
7631 A.isValid() && A.isStringAttribute()) {
7632 F.setSection(A.getValueAsString());
7633 AttrsToRemove.addAttribute("implicit-section-name");
7634 RemovingAttrs = true;
7635 }
7636
7637 if (Attribute A = F.getFnAttribute("nooutline");
7638 A.isValid() && A.isStringAttribute()) {
7639 AttrsToRemove.addAttribute("nooutline");
7640 AttrsToAdd.addAttribute(Attribute::NoOutline);
7641 AddingAttrs = RemovingAttrs = true;
7642 }
7643
7644 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7645 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7646 AttrsToRemove.addAttribute("uniform-work-group-size");
7647 RemovingAttrs = true;
7648 if (A.getValueAsString() == "true") {
7649 AttrsToAdd.addAttribute("uniform-work-group-size");
7650 AddingAttrs = true;
7651 }
7652 }
7653
7654 if (!F.empty()) {
7655 // For some reason this is called twice, and the first time is before any
7656 // instructions are loaded into the body.
7657
7658 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7659 A.isValid()) {
7660
7661 if (A.getValueAsBool()) {
7662 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7663 Visitor.visit(F);
7664 }
7665
7666 // We will leave behind dead attribute uses on external declarations, but
7667 // clang never added these to declarations anyway.
7668 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7669 RemovingAttrs = true;
7670 }
7671 }
7672
7673 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7674 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7675
7676 bool HandleDenormalMode = false;
7677
7678 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7679 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7680 if (ParsedMode.isValid()) {
7681 DenormalFPMath = ParsedMode;
7682 AttrsToRemove.addAttribute("denormal-fp-math");
7683 AddingAttrs = RemovingAttrs = true;
7684 HandleDenormalMode = true;
7685 }
7686 }
7687
7688 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7689 Attr.isValid()) {
7690 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7691 if (ParsedMode.isValid()) {
7692 DenormalFPMathF32 = ParsedMode;
7693 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7694 AddingAttrs = RemovingAttrs = true;
7695 HandleDenormalMode = true;
7696 }
7697 }
7698
7699 if (HandleDenormalMode)
7700 AttrsToAdd.addDenormalFPEnvAttr(
7701 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7702
7703 if (RemovingAttrs)
7704 F.removeFnAttrs(AttrsToRemove);
7705
7706 if (AddingAttrs)
7707 F.addFnAttrs(AttrsToAdd);
7708}
7709
7710// Check if the function attribute is not present and set it.
7711static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7712 StringRef Value) {
7713 if (!F.hasFnAttribute(FnAttrName)) {
7714 F.addFnAttr(FnAttrName, Value);
7715 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7716 << "\", function: " << F.getName() << "\n");
7717 }
7718}
7719
7720// Check if the function attribute is not present and set it if needed.
7721// If the attribute is "false" then removes it.
7722// If the attribute is "true" resets it to a valueless attribute.
7723static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7724 if (!F.hasFnAttribute(FnAttrName)) {
7725 if (Set) {
7726 F.addFnAttr(FnAttrName);
7727 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7728 << ", function: " << F.getName() << "\n");
7729 }
7730 } else {
7731 auto A = F.getFnAttribute(FnAttrName);
7732 if ("false" == A.getValueAsString()) {
7733 F.removeFnAttr(FnAttrName);
7734 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7735 << "=\"false\", function: " << F.getName() << "\n");
7736 } else if ("true" == A.getValueAsString()) {
7737 F.removeFnAttr(FnAttrName);
7738 F.addFnAttr(FnAttrName);
7739 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7740 << "=\"true\", function: " << F.getName() << "\n");
7741 }
7742 }
7743}
7744
7745static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7746 StringRef Key, uint32_t Val) {
7747 M.setModuleFlag(Behavior, Key, Val);
7748 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7749 << Key << ", " << Val << "}\n");
7750}
7751
7752void llvm::copyModuleAttrToFunctions(Module &M) {
7753 Triple T(M.getTargetTriple());
7754 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7755 return;
7756
7757 uint64_t BTEValue = 0;
7758 uint64_t BPPLRValue = 0;
7759 uint64_t GCSValue = 0;
7760 uint64_t SRAValue = 0;
7761 uint64_t SRAALLValue = 0;
7762 uint64_t SRABKeyValue = 0;
7763
7764 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7765 if (ModFlags) {
7766 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7767 MDNode *Op = ModFlags->getOperand(I);
7768 if (Op->getNumOperands() != 3)
7769 continue;
7770
7771 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7772 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7773 if (!ID || !CI)
7774 continue;
7775
7776 StringRef IDStr = ID->getString();
7777 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7778 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7779 : IDStr == "guarded-control-stack" ? &GCSValue
7780 : IDStr == "sign-return-address" ? &SRAValue
7781 : IDStr == "sign-return-address-all" ? &SRAALLValue
7782 : IDStr == "sign-return-address-with-bkey"
7783 ? &SRABKeyValue
7784 : nullptr;
7785 if (!ValPtr)
7786 continue;
7787
7788 *ValPtr = CI->getZExtValue();
7789 if (*ValPtr == 2)
7790 return;
7791
7792 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7793 << ")\n");
7794 }
7795 }
7796
7797 bool BTE = BTEValue == 1;
7798 bool BPPLR = BPPLRValue == 1;
7799 bool GCS = GCSValue == 1;
7800 bool SRA = SRAValue == 1;
7801
7802 StringRef SignTypeValue = "non-leaf";
7803 if (SRA && SRAALLValue == 1)
7804 SignTypeValue = "all";
7805
7806 StringRef SignKeyValue = "a_key";
7807 if (SRA && SRABKeyValue == 1)
7808 SignKeyValue = "b_key";
7809
7810 for (Function &F : M.getFunctionList()) {
7811 if (F.isDeclaration())
7812 continue;
7813
7814 if (SRA) {
7815 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7816 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7817 } else {
7818 if (auto A = F.getFnAttribute("sign-return-address");
7819 A.isValid() && "none" == A.getValueAsString()) {
7820 F.removeFnAttr("sign-return-address");
7821 F.removeFnAttr("sign-return-address-key");
7822 }
7823 }
7824 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7825 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7826 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7827 }
7828
7829 if (BTE)
7830 ConvertModuleFlag(M, llvm::Module::Min, "branch-target-enforcement", 2);
7831 if (BPPLR)
7832 ConvertModuleFlag(M, llvm::Module::Min, "branch-protection-pauth-lr", 2);
7833 if (GCS)
7834 ConvertModuleFlag(M, llvm::Module::Min, "guarded-control-stack", 2);
7835 if (SRA) {
7836 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address", 2);
7837 if (SRAALLValue == 1)
7838 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-all", 2);
7839 if (SRABKeyValue == 1) {
7840 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-with-bkey",
7841 2);
7842 }
7843 }
7844}
7845
7846/// Return the replacement tags if \p T still uses a removed two-operand form.
7847static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7848 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7849 return nullptr;
7850 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7851 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7852}
7853
7854/// Build the single-operand node that replaces a boolean operand: nonzero
7855/// selects the enable tag, zero the disable tag.
7856static Metadata *makeBooleanLoopNode(LLVMContext &C,
7857 const BooleanLoopTags &Tags,
7858 const MDOperand &Op) {
7859 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7860 return MDTuple::get(C,
7861 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7862}
7863
7864static bool isOldLoopArgument(Metadata *MD) {
7865 auto *T = dyn_cast_or_null<MDTuple>(MD);
7866 if (!T)
7867 return false;
7868 if (T->getNumOperands() < 1)
7869 return false;
7870 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7871 if (!S)
7872 return false;
7873 if (S->getString().starts_with("llvm.vectorizer."))
7874 return true;
7875 return getOldBooleanLoopTags(T) != nullptr;
7876}
7877
7878static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7879 StringRef OldPrefix = "llvm.vectorizer.";
7880 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7881
7882 if (OldTag == "llvm.vectorizer.unroll")
7883 return MDString::get(C, "llvm.loop.interleave.count");
7884
7885 return MDString::get(
7886 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7887 .str());
7888}
7889
7890static Metadata *upgradeLoopArgument(Metadata *MD) {
7891 auto *T = dyn_cast_or_null<MDTuple>(MD);
7892 if (!T)
7893 return MD;
7894 if (T->getNumOperands() < 1)
7895 return MD;
7896 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7897 if (!OldTag)
7898 return MD;
7899
7900 LLVMContext &C = T->getContext();
7901
7902 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7903 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7904 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7905
7906 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7907 return MD;
7908
7909 // This has an old tag. Upgrade it.
7910 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7911
7912 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7913 // vectorize.enable/disable pair, not a two-operand enable node.
7914 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7915 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7916 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7917
7918 SmallVector<Metadata *, 8> Ops;
7919 Ops.reserve(T->getNumOperands());
7920 Ops.push_back(NewTag);
7921 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7922 Ops.push_back(T->getOperand(I));
7923
7924 return MDTuple::get(C, Ops);
7925}
7926
7927MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7928 auto *T = dyn_cast<MDTuple>(&N);
7929 if (!T)
7930 return &N;
7931
7932 if (none_of(T->operands(), isOldLoopArgument))
7933 return &N;
7934
7935 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7936 // any MDNode carrying those tags with more than one operand, so a leftover
7937 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7938 // In-place mutation is safe on distinct MDNodes.
7939 if (T->isDistinct()) {
7940 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7941 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7942 if (OpT && getOldBooleanLoopTags(OpT))
7943 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7944 }
7945 if (none_of(T->operands(), isOldLoopArgument))
7946 return &N;
7947 }
7948
7949 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7950 // attachment; the original distinct loop-ID is kept as the first operand.
7951 SmallVector<Metadata *, 8> Ops;
7952 Ops.reserve(T->getNumOperands());
7953 for (Metadata *MD : T->operands())
7954 Ops.push_back(upgradeLoopArgument(MD));
7955
7956 return MDTuple::get(T->getContext(), Ops);
7957}
7958
7959std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7960 Triple T(TT);
7961 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7962 // the address space of globals to 1. This does not apply to SPIRV Logical.
7963 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7964 !DL.contains("-G") && !DL.starts_with("G")) {
7965 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7966 }
7967
7968 if (T.isLoongArch64() || T.isRISCV64()) {
7969 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7970 auto I = DL.find("-n64-");
7971 if (I != StringRef::npos)
7972 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7973 return DL.str();
7974 }
7975
7976 // AMDGPU data layout upgrades.
7977 std::string Res = DL.str();
7978 if (T.isAMDGPU()) {
7979 // Define address spaces for constants.
7980 if (!DL.contains("-G") && !DL.starts_with("G"))
7981 Res.append(Res.empty() ? "G1" : "-G1");
7982
7983 // AMDGCN data layout upgrades.
7984 if (T.isAMDGCN()) {
7985
7986 // Add missing non-integral declarations.
7987 // This goes before adding new address spaces to prevent incoherent string
7988 // values.
7989 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7990 Res.append("-ni:7:8:9");
7991 // Update ni:7 to ni:7:8:9.
7992 if (DL.ends_with("ni:7"))
7993 Res.append(":8:9");
7994 if (DL.ends_with("ni:7:8"))
7995 Res.append(":9");
7996
7997 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7998 // resources) An empty data layout has already been upgraded to G1 by now.
7999 if (!DL.contains("-p7") && !DL.starts_with("p7"))
8000 Res.append("-p7:160:256:256:32");
8001 if (!DL.contains("-p8") && !DL.starts_with("p8"))
8002 Res.append("-p8:128:128:128:48");
8003 constexpr StringRef OldP8("-p8:128:128-");
8004 if (DL.contains(OldP8))
8005 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
8006 if (!DL.contains("-p9") && !DL.starts_with("p9"))
8007 Res.append("-p9:192:256:256:32");
8008
8009 // Add sizing for address space 10 through 15.
8010 // AS 10-14 are reserved and defaulted to 32:32
8011 // AS 15 is in use and is 32:32.
8012 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
8013 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
8014 Res.append(("-" + AS + ":32:32").str());
8015 }
8016 }
8017
8018 // Upgrade the ELF mangling mode.
8019 if (!DL.contains("m:e"))
8020 Res = Res.empty() ? "m:e" : "m:e-" + Res;
8021
8022 return Res;
8023 }
8024
8025 if (T.isSystemZ() && !DL.empty()) {
8026 // Make sure the stack alignment is present.
8027 if (!DL.contains("-S64"))
8028 return "E-S64" + DL.drop_front(1).str();
8029 return DL.str();
8030 }
8031
8032 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
8033 // If the datalayout matches the expected format, add pointer size address
8034 // spaces to the datalayout.
8035 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
8036 if (!DL.contains(AddrSpaces)) {
8037 SmallVector<StringRef, 4> Groups;
8038 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
8039 if (R.match(Res, &Groups))
8040 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
8041 }
8042 };
8043
8044 // AArch64 data layout upgrades.
8045 if (T.isAArch64()) {
8046 // Add "-Fn32"
8047 if (!DL.empty() && !DL.contains("-Fn32"))
8048 Res.append("-Fn32");
8049 AddPtr32Ptr64AddrSpaces();
8050 return Res;
8051 }
8052
8053 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
8054 T.isWasm()) {
8055 // Mips64 with o32 ABI did not add "-i128:128".
8056 // Add "-i128:128"
8057 std::string I64 = "-i64:64";
8058 std::string I128 = "-i128:128";
8059 if (!StringRef(Res).contains(I128)) {
8060 size_t Pos = Res.find(I64);
8061 if (Pos != size_t(-1))
8062 Res.insert(Pos + I64.size(), I128);
8063 }
8064 }
8065
8066 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
8067 size_t Pos = Res.find("-S128");
8068 if (Pos == StringRef::npos)
8069 Pos = Res.size();
8070 Res.insert(Pos, "-f64:32:64");
8071 }
8072
8073 // ARM data layout upgrades.
8074 // Add -Fi8 if a -F has not already been specified.
8075 if (T.isARM() && !DL.empty() && !DL.contains("Fi") && !DL.contains("Fn")) {
8076 const StringRef p3232 = "p:32:32";
8077 size_t Pos = Res.find(p3232);
8078 if (Pos != StringRef::npos)
8079 Res.insert(Pos + p3232.size(), "-Fi8");
8080 }
8081
8082 if (!T.isX86())
8083 return Res;
8084
8085 AddPtr32Ptr64AddrSpaces();
8086
8087 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
8088 // for i128 operations prior to this being reflected in the data layout, and
8089 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
8090 // boundaries, so although this is a breaking change, the upgrade is expected
8091 // to fix more IR than it breaks.
8092 // Intel MCU is an exception and uses 4-byte-alignment.
8093 if (!T.isOSIAMCU()) {
8094 std::string I128 = "-i128:128";
8095 if (StringRef Ref = Res; !Ref.contains(I128)) {
8096 SmallVector<StringRef, 4> Groups;
8097 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8098 if (R.match(Res, &Groups))
8099 Res = (Groups[1] + I128 + Groups[3]).str();
8100 }
8101 }
8102
8103 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
8104 // Raising the alignment is safe because Clang did not produce f80 values in
8105 // the MSVC environment before this upgrade was added.
8106 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8107 StringRef Ref = Res;
8108 auto I = Ref.find("-f80:32-");
8109 if (I != StringRef::npos)
8110 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
8111 }
8112
8113 return Res;
8114}
8115
8116void llvm::UpgradeAttributes(AttrBuilder &B) {
8117 StringRef FramePointer;
8118 Attribute A = B.getAttribute("no-frame-pointer-elim");
8119 if (A.isValid()) {
8120 // The value can be "true" or "false".
8121 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8122 B.removeAttribute("no-frame-pointer-elim");
8123 }
8124 if (B.contains("no-frame-pointer-elim-non-leaf")) {
8125 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8126 if (FramePointer != "all")
8127 FramePointer = "non-leaf";
8128 B.removeAttribute("no-frame-pointer-elim-non-leaf");
8129 }
8130 if (!FramePointer.empty())
8131 B.addAttribute("frame-pointer", FramePointer);
8132
8133 A = B.getAttribute("null-pointer-is-valid");
8134 if (A.isValid()) {
8135 // The value can be "true" or "false".
8136 bool NullPointerIsValid = A.getValueAsString() == "true";
8137 B.removeAttribute("null-pointer-is-valid");
8138 if (NullPointerIsValid)
8139 B.addAttribute(Attribute::NullPointerIsValid);
8140 }
8141
8142 A = B.getAttribute("uniform-work-group-size");
8143 if (A.isValid()) {
8144 StringRef Val = A.getValueAsString();
8145 if (!Val.empty()) {
8146 bool IsTrue = Val == "true";
8147 B.removeAttribute("uniform-work-group-size");
8148 if (IsTrue)
8149 B.addAttribute("uniform-work-group-size");
8150 }
8151 }
8152}
8153
8154void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8155 // clang.arc.attachedcall bundles are now required to have an operand.
8156 // If they don't, it's okay to drop them entirely: when there is an operand,
8157 // the "attachedcall" is meaningful and required, but without an operand,
8158 // it's just a marker NOP. Dropping it merely prevents an optimization.
8159 erase_if(Bundles, [&](OperandBundleDef &OBD) {
8160 return OBD.getTag() == "clang.arc.attachedcall" &&
8161 OBD.inputs().empty();
8162 });
8163}
Pred
SDValue Pred
Definition AArch64ISelLowering.cpp:28622
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
Imm
uint64_t Imm
Definition AMDGPUDisassembler.cpp:552
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2738
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1503
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
Definition AutoUpgrade.cpp:1709
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7890
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1591
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6958
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1745
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2578
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1386
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5756
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1339
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1352
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:7064
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2839
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3065
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1688
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2813
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:125
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1641
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7711
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1445
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6962
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5616
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7878
isValidLegacyNVVMFPArithIntrinsic
static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
Definition AutoUpgrade.cpp:1541
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5716
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6925
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:112
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2693
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2950
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1684
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2664
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5253
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3051
NVVMFMAIIDs
static constexpr Intrinsic::ID NVVMFMAIIDs[2][3]
Definition AutoUpgrade.cpp:1534
NVVMFMulIIDs
static constexpr Intrinsic::ID NVVMFMulIIDs[2][3]
Definition AutoUpgrade.cpp:1530
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7847
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:556
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2990
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6871
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1598
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7856
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:179
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2612
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1604
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1219
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1410
shouldUpgradeNVVMBarrierMulticastIntrinsic
static Intrinsic::ID shouldUpgradeNVVMBarrierMulticastIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1373
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7864
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5323
ConvertModuleFlag
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
Definition AutoUpgrade.cpp:7745
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:96
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5686
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5413
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2929
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5628
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:159
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:794
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5787
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2644
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2676
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1729
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2878
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:67
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:80
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:169
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1194
upgradeNVVMFPArithCall
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:3314
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:3012
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2798
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3340
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1295
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:3044
shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1435
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:3074
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5638
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7723
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2960
getNVVMFPArithUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:1558
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:71
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2911
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1323
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1262
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5699
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:143
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7219
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3555
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5734
NVVMFAddIIDs
static constexpr Intrinsic::ID NVVMFAddIIDs[2][3]
Definition AutoUpgrade.cpp:1526
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:268
T1
#define T1
Definition Mips16ISelLowering.cpp:267
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
C1
int64_t C1
Definition RISCVLateBranchOpt.cpp:64
CC
RISCVCC::CondCode CC
Definition RISCVLateBranchOpt.cpp:57
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:774
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:934
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:786
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:790
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:811
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:850
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:794
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:800
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:854
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:842
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:802
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:822
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:818
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:846
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:981
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:508
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:356
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1595
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1715
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3198
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3481
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1830
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1454
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2521
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2453
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2507
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:2034
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1521
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1685
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:44
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:449
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:389
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3621
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2445
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:728
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:571
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2987
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1788
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:697
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1543
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:320
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::op_end
op_iterator op_end() const
Definition Metadata.h:1431
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::op_begin
op_iterator op_begin() const
Definition Metadata.h:1427
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:595
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1496
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1525
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:105
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1797
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1592
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1584
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1580
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1599
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1893
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1590
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2092
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2095
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:594
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:720
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:425
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1261
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::find
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringRef::consume_front
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:194
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5372
llvm::Value::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Definition Metadata.cpp:1670
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1451
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:637
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1423
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1416
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5848
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7536
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3305
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:201
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2527
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7927
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:8116
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6762
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:7023
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7318
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7492
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7752
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:8154
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6851
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7959
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::UpgradeTBAAStructNode
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
Definition AutoUpgrade.cpp:6807
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2546
llvm::Key
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Definition PassManager.h:695
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:718
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:134
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6829
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getIROptions
const IROptions & getIROptions(const LLVMContext &C)
Definition IROptions.h:27
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6881
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7609
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6781
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:7088
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:8173
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:120
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:122
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:121
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
Generated on for LLVM by doxygen 1.14.0