LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstVisitor.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsAArch64.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsARM.h"
39#include "llvm/IR/IntrinsicsNVPTX.h"
40#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/IR/IntrinsicsWebAssembly.h"
42#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/MDBuilder.h"
45#include "llvm/IR/Metadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/NVVMIntrinsicUtils.h"
48#include "llvm/IR/Value.h"
49#include "llvm/IR/Verifier.h"
50#include "llvm/Support/AMDGPUAddrSpace.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/NVPTXAddrSpace.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/Regex.h"
57#include "llvm/Support/TimeProfiler.h"
58#include "llvm/TargetParser/Triple.h"
59#include <cstdint>
60#include <cstring>
61#include <numeric>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "auto-upgrade"
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(0);
85 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(F);
90 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(F);
106 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(F);
119 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
134 return false;
135
136 rename(F);
137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
152 return false;
153
154 rename(F);
155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(F);
165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(F);
175 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front("avx."))
186 return (Name.starts_with("blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with("movnt.") || // Added in 3.2
191 Name.starts_with("sqrt.p") || // Added in 7.0
192 Name.starts_with("storeu.") || // Added in 3.9
193 Name.starts_with("vbroadcast.s") || // Added in 3.5
194 Name.starts_with("vbroadcastf128") || // Added in 4.0
195 Name.starts_with("vextractf128.") || // Added in 3.7
196 Name.starts_with("vinsertf128.") || // Added in 3.7
197 Name.starts_with("vperm2f128.") || // Added in 6.0
198 Name.starts_with("vpermil.")); // Added in 3.1
199
200 if (Name.consume_front("avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with("pabs.") || // Added in 6.0
203 Name.starts_with("padds.") || // Added in 8.0
204 Name.starts_with("paddus.") || // Added in 8.0
205 Name.starts_with("pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with("pbroadcast") || // Added in 3.8
208 Name.starts_with("pcmpeq.") || // Added in 3.1
209 Name.starts_with("pcmpgt.") || // Added in 3.1
210 Name.starts_with("pmax") || // Added in 3.9
211 Name.starts_with("pmin") || // Added in 3.9
212 Name.starts_with("pmovsx") || // Added in 3.9
213 Name.starts_with("pmovzx") || // Added in 3.9
214 Name.starts_with("pmulh.w") || // Added in 24.0
215 Name.starts_with("pmulhu.w") || // Added in 24.0
216 Name == "pmul.dq" || // Added in 7.0
217 Name == "pmulu.dq" || // Added in 7.0
218 Name.starts_with("psll.dq") || // Added in 3.7
219 Name.starts_with("psrl.dq") || // Added in 3.7
220 Name.starts_with("psubs.") || // Added in 8.0
221 Name.starts_with("psubus.") || // Added in 8.0
222 Name.starts_with("vbroadcast") || // Added in 3.8
223 Name == "vbroadcasti128" || // Added in 3.7
224 Name == "vextracti128" || // Added in 3.7
225 Name == "vinserti128" || // Added in 3.7
226 Name == "vperm2i128"); // Added in 6.0
227
228 if (Name.consume_front("avx512.")) {
229 if (Name.consume_front("mask."))
230 // 'avx512.mask.*'
231 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
232 Name.starts_with("and.") || // Added in 3.9
233 Name.starts_with("andn.") || // Added in 3.9
234 Name.starts_with("broadcast.s") || // Added in 3.9
235 Name.starts_with("broadcastf32x4.") || // Added in 6.0
236 Name.starts_with("broadcastf32x8.") || // Added in 6.0
237 Name.starts_with("broadcastf64x2.") || // Added in 6.0
238 Name.starts_with("broadcastf64x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x4.") || // Added in 6.0
240 Name.starts_with("broadcasti32x8.") || // Added in 6.0
241 Name.starts_with("broadcasti64x2.") || // Added in 6.0
242 Name.starts_with("broadcasti64x4.") || // Added in 6.0
243 Name.starts_with("cmp.b") || // Added in 5.0
244 Name.starts_with("cmp.d") || // Added in 5.0
245 Name.starts_with("cmp.q") || // Added in 5.0
246 Name.starts_with("cmp.w") || // Added in 5.0
247 Name.starts_with("compress.b") || // Added in 9.0
248 Name.starts_with("compress.d") || // Added in 9.0
249 Name.starts_with("compress.p") || // Added in 9.0
250 Name.starts_with("compress.q") || // Added in 9.0
251 Name.starts_with("compress.store.") || // Added in 7.0
252 Name.starts_with("compress.w") || // Added in 9.0
253 Name.starts_with("conflict.") || // Added in 9.0
254 Name.starts_with("cvtdq2pd.") || // Added in 4.0
255 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
256 Name == "cvtpd2dq.256" || // Added in 7.0
257 Name == "cvtpd2ps.256" || // Added in 7.0
258 Name == "cvtps2pd.128" || // Added in 7.0
259 Name == "cvtps2pd.256" || // Added in 7.0
260 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
261 Name == "cvtqq2ps.256" || // Added in 9.0
262 Name == "cvtqq2ps.512" || // Added in 9.0
263 Name == "cvttpd2dq.256" || // Added in 7.0
264 Name == "cvttps2dq.128" || // Added in 7.0
265 Name == "cvttps2dq.256" || // Added in 7.0
266 Name.starts_with("cvtudq2pd.") || // Added in 4.0
267 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
268 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
269 Name == "cvtuqq2ps.256" || // Added in 9.0
270 Name == "cvtuqq2ps.512" || // Added in 9.0
271 Name.starts_with("dbpsadbw.") || // Added in 7.0
272 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
273 Name.starts_with("expand.b") || // Added in 9.0
274 Name.starts_with("expand.d") || // Added in 9.0
275 Name.starts_with("expand.load.") || // Added in 7.0
276 Name.starts_with("expand.p") || // Added in 9.0
277 Name.starts_with("expand.q") || // Added in 9.0
278 Name.starts_with("expand.w") || // Added in 9.0
279 Name.starts_with("fpclass.p") || // Added in 7.0
280 Name.starts_with("insert") || // Added in 4.0
281 Name.starts_with("load.") || // Added in 3.9
282 Name.starts_with("loadu.") || // Added in 3.9
283 Name.starts_with("lzcnt.") || // Added in 5.0
284 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with("movddup") || // Added in 3.9
287 Name.starts_with("move.s") || // Added in 4.0
288 Name.starts_with("movshdup") || // Added in 3.9
289 Name.starts_with("movsldup") || // Added in 3.9
290 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
291 Name.starts_with("or.") || // Added in 3.9
292 Name.starts_with("pabs.") || // Added in 6.0
293 Name.starts_with("packssdw.") || // Added in 5.0
294 Name.starts_with("packsswb.") || // Added in 5.0
295 Name.starts_with("packusdw.") || // Added in 5.0
296 Name.starts_with("packuswb.") || // Added in 5.0
297 Name.starts_with("padd.") || // Added in 4.0
298 Name.starts_with("padds.") || // Added in 8.0
299 Name.starts_with("paddus.") || // Added in 8.0
300 Name.starts_with("palignr.") || // Added in 3.9
301 Name.starts_with("pand.") || // Added in 3.9
302 Name.starts_with("pandn.") || // Added in 3.9
303 Name.starts_with("pavg") || // Added in 6.0
304 Name.starts_with("pbroadcast") || // Added in 6.0
305 Name.starts_with("pcmpeq.") || // Added in 3.9
306 Name.starts_with("pcmpgt.") || // Added in 3.9
307 Name.starts_with("perm.df.") || // Added in 3.9
308 Name.starts_with("perm.di.") || // Added in 3.9
309 Name.starts_with("permvar.") || // Added in 7.0
310 Name.starts_with("pmaddubs.w.") || // Added in 7.0
311 Name.starts_with("pmaddw.d.") || // Added in 7.0
312 Name.starts_with("pmax") || // Added in 4.0
313 Name.starts_with("pmin") || // Added in 4.0
314 Name == "pmov.qd.256" || // Added in 9.0
315 Name == "pmov.qd.512" || // Added in 9.0
316 Name == "pmov.wb.256" || // Added in 9.0
317 Name == "pmov.wb.512" || // Added in 9.0
318 Name.starts_with("pmovsx") || // Added in 4.0
319 Name.starts_with("pmovzx") || // Added in 4.0
320 Name.starts_with("pmul.dq.") || // Added in 4.0
321 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
322 Name.starts_with("pmulh.w.") || // Added in 7.0
323 Name.starts_with("pmulhu.w.") || // Added in 7.0
324 Name.starts_with("pmull.") || // Added in 4.0
325 Name.starts_with("pmultishift.qb.") || // Added in 8.0
326 Name.starts_with("pmulu.dq.") || // Added in 4.0
327 Name.starts_with("por.") || // Added in 3.9
328 Name.starts_with("prol.") || // Added in 8.0
329 Name.starts_with("prolv.") || // Added in 8.0
330 Name.starts_with("pror.") || // Added in 8.0
331 Name.starts_with("prorv.") || // Added in 8.0
332 Name.starts_with("pshuf.b.") || // Added in 4.0
333 Name.starts_with("pshuf.d.") || // Added in 3.9
334 Name.starts_with("pshufh.w.") || // Added in 3.9
335 Name.starts_with("pshufl.w.") || // Added in 3.9
336 Name.starts_with("psll.d") || // Added in 4.0
337 Name.starts_with("psll.q") || // Added in 4.0
338 Name.starts_with("psll.w") || // Added in 4.0
339 Name.starts_with("pslli") || // Added in 4.0
340 Name.starts_with("psllv") || // Added in 4.0
341 Name.starts_with("psra.d") || // Added in 4.0
342 Name.starts_with("psra.q") || // Added in 4.0
343 Name.starts_with("psra.w") || // Added in 4.0
344 Name.starts_with("psrai") || // Added in 4.0
345 Name.starts_with("psrav") || // Added in 4.0
346 Name.starts_with("psrl.d") || // Added in 4.0
347 Name.starts_with("psrl.q") || // Added in 4.0
348 Name.starts_with("psrl.w") || // Added in 4.0
349 Name.starts_with("psrli") || // Added in 4.0
350 Name.starts_with("psrlv") || // Added in 4.0
351 Name.starts_with("psub.") || // Added in 4.0
352 Name.starts_with("psubs.") || // Added in 8.0
353 Name.starts_with("psubus.") || // Added in 8.0
354 Name.starts_with("pternlog.") || // Added in 7.0
355 Name.starts_with("punpckh") || // Added in 3.9
356 Name.starts_with("punpckl") || // Added in 3.9
357 Name.starts_with("pxor.") || // Added in 3.9
358 Name.starts_with("shuf.f") || // Added in 6.0
359 Name.starts_with("shuf.i") || // Added in 6.0
360 Name.starts_with("shuf.p") || // Added in 4.0
361 Name.starts_with("sqrt.p") || // Added in 7.0
362 Name.starts_with("store.b.") || // Added in 3.9
363 Name.starts_with("store.d.") || // Added in 3.9
364 Name.starts_with("store.p") || // Added in 3.9
365 Name.starts_with("store.q.") || // Added in 3.9
366 Name.starts_with("store.w.") || // Added in 3.9
367 Name == "store.ss" || // Added in 7.0
368 Name.starts_with("storeu.") || // Added in 3.9
369 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
370 Name.starts_with("ucmp.") || // Added in 5.0
371 Name.starts_with("unpckh.") || // Added in 3.9
372 Name.starts_with("unpckl.") || // Added in 3.9
373 Name.starts_with("valign.") || // Added in 4.0
374 Name == "vcvtph2ps.128" || // Added in 11.0
375 Name == "vcvtph2ps.256" || // Added in 11.0
376 Name.starts_with("vextract") || // Added in 4.0
377 Name.starts_with("vfmadd.") || // Added in 7.0
378 Name.starts_with("vfmaddsub.") || // Added in 7.0
379 Name.starts_with("vfnmadd.") || // Added in 7.0
380 Name.starts_with("vfnmsub.") || // Added in 7.0
381 Name.starts_with("vpdpbusd.") || // Added in 7.0
382 Name.starts_with("vpdpbusds.") || // Added in 7.0
383 Name.starts_with("vpdpwssd.") || // Added in 7.0
384 Name.starts_with("vpdpwssds.") || // Added in 7.0
385 Name.starts_with("vpermi2var.") || // Added in 7.0
386 Name.starts_with("vpermil.p") || // Added in 3.9
387 Name.starts_with("vpermilvar.") || // Added in 4.0
388 Name.starts_with("vpermt2var.") || // Added in 7.0
389 Name.starts_with("vpmadd52") || // Added in 7.0
390 Name.starts_with("vpshld.") || // Added in 7.0
391 Name.starts_with("vpshldv.") || // Added in 8.0
392 Name.starts_with("vpshrd.") || // Added in 7.0
393 Name.starts_with("vpshrdv.") || // Added in 8.0
394 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
395 Name.starts_with("xor.")); // Added in 3.9
396
397 if (Name.consume_front("mask3."))
398 // 'avx512.mask3.*'
399 return (Name.starts_with("vfmadd.") || // Added in 7.0
400 Name.starts_with("vfmaddsub.") || // Added in 7.0
401 Name.starts_with("vfmsub.") || // Added in 7.0
402 Name.starts_with("vfmsubadd.") || // Added in 7.0
403 Name.starts_with("vfnmsub.")); // Added in 7.0
404
405 if (Name.consume_front("maskz."))
406 // 'avx512.maskz.*'
407 return (Name.starts_with("pternlog.") || // Added in 7.0
408 Name.starts_with("vfmadd.") || // Added in 7.0
409 Name.starts_with("vfmaddsub.") || // Added in 7.0
410 Name.starts_with("vpdpbusd.") || // Added in 7.0
411 Name.starts_with("vpdpbusds.") || // Added in 7.0
412 Name.starts_with("vpdpwssd.") || // Added in 7.0
413 Name.starts_with("vpdpwssds.") || // Added in 7.0
414 Name.starts_with("vpermt2var.") || // Added in 7.0
415 Name.starts_with("vpmadd52") || // Added in 7.0
416 Name.starts_with("vpshldv.") || // Added in 8.0
417 Name.starts_with("vpshrdv.")); // Added in 8.0
418
419 // 'avx512.*'
420 return (Name == "movntdqa" || // Added in 5.0
421 Name == "pmul.dq.512" || // Added in 7.0
422 Name == "pmulu.dq.512" || // Added in 7.0
423 Name.starts_with("broadcastm") || // Added in 6.0
424 Name.starts_with("cmp.p") || // Added in 12.0
425 Name.starts_with("cvtb2mask.") || // Added in 7.0
426 Name.starts_with("cvtd2mask.") || // Added in 7.0
427 Name.starts_with("cvtmask2") || // Added in 5.0
428 Name.starts_with("cvtq2mask.") || // Added in 7.0
429 Name == "cvtusi2sd" || // Added in 7.0
430 Name.starts_with("cvtw2mask.") || // Added in 7.0
431 Name == "kand.w" || // Added in 7.0
432 Name == "kandn.w" || // Added in 7.0
433 Name == "knot.w" || // Added in 7.0
434 Name == "kor.w" || // Added in 7.0
435 Name == "kortestc.w" || // Added in 7.0
436 Name == "kortestz.w" || // Added in 7.0
437 Name.starts_with("kunpck") || // added in 6.0
438 Name == "kxnor.w" || // Added in 7.0
439 Name == "kxor.w" || // Added in 7.0
440 Name.starts_with("padds.") || // Added in 8.0
441 Name.starts_with("pbroadcast") || // Added in 3.9
442 Name.starts_with("pmulh.w") || // Added in 24.0
443 Name.starts_with("pmulhu.w") || // Added in 24.0
444 Name.starts_with("prol") || // Added in 8.0
445 Name.starts_with("pror") || // Added in 8.0
446 Name.starts_with("psll.dq") || // Added in 3.9
447 Name.starts_with("psrl.dq") || // Added in 3.9
448 Name.starts_with("psubs.") || // Added in 8.0
449 Name.starts_with("ptestm") || // Added in 6.0
450 Name.starts_with("ptestnm") || // Added in 6.0
451 Name.starts_with("storent.") || // Added in 3.9
452 Name.starts_with("vbroadcast.s") || // Added in 7.0
453 Name.starts_with("vpshld.") || // Added in 8.0
454 Name.starts_with("vpshrd.")); // Added in 8.0
455 }
456
457 if (Name.consume_front("fma."))
458 return (Name.starts_with("vfmadd.") || // Added in 7.0
459 Name.starts_with("vfmsub.") || // Added in 7.0
460 Name.starts_with("vfmsubadd.") || // Added in 7.0
461 Name.starts_with("vfnmadd.") || // Added in 7.0
462 Name.starts_with("vfnmsub.")); // Added in 7.0
463
464 if (Name.consume_front("fma4."))
465 return Name.starts_with("vfmadd.s"); // Added in 7.0
466
467 if (Name.consume_front("sse."))
468 return (Name == "add.ss" || // Added in 4.0
469 Name == "cvtsi2ss" || // Added in 7.0
470 Name == "cvtsi642ss" || // Added in 7.0
471 Name == "div.ss" || // Added in 4.0
472 Name == "mul.ss" || // Added in 4.0
473 Name.starts_with("sqrt.p") || // Added in 7.0
474 Name == "sqrt.ss" || // Added in 7.0
475 Name.starts_with("storeu.") || // Added in 3.9
476 Name == "sub.ss"); // Added in 4.0
477
478 if (Name.consume_front("sse2."))
479 return (Name == "add.sd" || // Added in 4.0
480 Name == "cvtdq2pd" || // Added in 3.9
481 Name == "cvtdq2ps" || // Added in 7.0
482 Name == "cvtps2pd" || // Added in 3.9
483 Name == "cvtsi2sd" || // Added in 7.0
484 Name == "cvtsi642sd" || // Added in 7.0
485 Name == "cvtss2sd" || // Added in 7.0
486 Name == "div.sd" || // Added in 4.0
487 Name == "mul.sd" || // Added in 4.0
488 Name.starts_with("padds.") || // Added in 8.0
489 Name.starts_with("paddus.") || // Added in 8.0
490 Name.starts_with("pcmpeq.") || // Added in 3.1
491 Name.starts_with("pcmpgt.") || // Added in 3.1
492 Name == "pmaxs.w" || // Added in 3.9
493 Name == "pmaxu.b" || // Added in 3.9
494 Name == "pmins.w" || // Added in 3.9
495 Name == "pminu.b" || // Added in 3.9
496 Name == "pmulh.w" || // Added in 24.0
497 Name == "pmulhu.w" || // Added in 24.0
498 Name == "pmulu.dq" || // Added in 7.0
499 Name.starts_with("pshuf") || // Added in 3.9
500 Name.starts_with("psll.dq") || // Added in 3.7
501 Name.starts_with("psrl.dq") || // Added in 3.7
502 Name.starts_with("psubs.") || // Added in 8.0
503 Name.starts_with("psubus.") || // Added in 8.0
504 Name.starts_with("sqrt.p") || // Added in 7.0
505 Name == "sqrt.sd" || // Added in 7.0
506 Name == "storel.dq" || // Added in 3.9
507 Name.starts_with("storeu.") || // Added in 3.9
508 Name == "sub.sd"); // Added in 4.0
509
510 if (Name.consume_front("sse41."))
511 return (Name.starts_with("blendp") || // Added in 3.7
512 Name == "movntdqa" || // Added in 5.0
513 Name == "pblendw" || // Added in 3.7
514 Name == "pmaxsb" || // Added in 3.9
515 Name == "pmaxsd" || // Added in 3.9
516 Name == "pmaxud" || // Added in 3.9
517 Name == "pmaxuw" || // Added in 3.9
518 Name == "pminsb" || // Added in 3.9
519 Name == "pminsd" || // Added in 3.9
520 Name == "pminud" || // Added in 3.9
521 Name == "pminuw" || // Added in 3.9
522 Name.starts_with("pmovsx") || // Added in 3.8
523 Name.starts_with("pmovzx") || // Added in 3.9
524 Name == "pmuldq"); // Added in 7.0
525
526 if (Name.consume_front("sse42."))
527 return Name == "crc32.64.8"; // Added in 3.4
528
529 if (Name.consume_front("sse4a."))
530 return Name.starts_with("movnt."); // Added in 3.9
531
532 if (Name.consume_front("ssse3."))
533 return (Name == "pabs.b.128" || // Added in 6.0
534 Name == "pabs.d.128" || // Added in 6.0
535 Name == "pabs.w.128"); // Added in 6.0
536
537 if (Name.consume_front("xop."))
538 return (Name == "vpcmov" || // Added in 3.8
539 Name == "vpcmov.256" || // Added in 5.0
540 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
541 Name.starts_with("vprot")); // Added in 8.0
542
543 if (Name.consume_front("bmi."))
544 return (Name.starts_with("pdep.") || // Added in 23.0
545 Name.starts_with("pext.")); // Added in 23.0
546
547 return (Name == "addcarry.u32" || // Added in 8.0
548 Name == "addcarry.u64" || // Added in 8.0
549 Name == "addcarryx.u32" || // Added in 8.0
550 Name == "addcarryx.u64" || // Added in 8.0
551 Name == "subborrow.u32" || // Added in 8.0
552 Name == "subborrow.u64" || // Added in 8.0
553 Name.starts_with("vcvtph2ps.")); // Added in 11.0
554}
555
556static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
557 Function *&NewFn) {
558 // Only handle intrinsics that start with "x86.".
559 if (!Name.consume_front("x86."))
560 return false;
561
562 if (shouldUpgradeX86Intrinsic(F, Name)) {
563 NewFn = nullptr;
564 return true;
565 }
566
567 if (Name == "rdtscp") { // Added in 8.0
568 // If this intrinsic has 0 operands, it's the new version.
569 if (F->getFunctionType()->getNumParams() == 0)
570 return false;
571
572 rename(F);
573 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
574 Intrinsic::x86_rdtscp);
575 return true;
576 }
577
578 Intrinsic::ID ID;
579
580 // SSE4.1 ptest functions may have an old signature.
581 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
582 ID = StringSwitch<Intrinsic::ID>(Name)
583 .Case("c", Intrinsic::x86_sse41_ptestc)
584 .Case("z", Intrinsic::x86_sse41_ptestz)
585 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
586 .Default(Intrinsic::not_intrinsic);
587 if (ID != Intrinsic::not_intrinsic)
588 return upgradePTESTIntrinsic(F, ID, NewFn);
589
590 return false;
591 }
592
593 // Several blend and other instructions with masks used the wrong number of
594 // bits.
595
596 // Added in 3.6
597 ID = StringSwitch<Intrinsic::ID>(Name)
598 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
599 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
600 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
601 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
602 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
603 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
604 .Default(Intrinsic::not_intrinsic);
605 if (ID != Intrinsic::not_intrinsic)
606 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
607
608 if (Name.consume_front("avx512.")) {
609 if (Name.consume_front("mask.cmp.")) {
610 // Added in 7.0
611 ID = StringSwitch<Intrinsic::ID>(Name)
612 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
613 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
614 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
615 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
616 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
617 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
618 .Default(Intrinsic::not_intrinsic);
619 if (ID != Intrinsic::not_intrinsic)
620 return upgradeX86MaskedFPCompare(F, ID, NewFn);
621 } else if (Name.starts_with("vpdpbusd.") ||
622 Name.starts_with("vpdpbusds.")) {
623 // Added in 21.1
624 ID = StringSwitch<Intrinsic::ID>(Name)
625 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
626 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
627 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
628 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
629 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
630 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
631 .Default(Intrinsic::not_intrinsic);
632 if (ID != Intrinsic::not_intrinsic)
633 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
634 } else if (Name.starts_with("vpdpwssd.") ||
635 Name.starts_with("vpdpwssds.")) {
636 // Added in 21.1
637 ID = StringSwitch<Intrinsic::ID>(Name)
638 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
639 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
640 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
641 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
642 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
643 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
644 .Default(Intrinsic::not_intrinsic);
645 if (ID != Intrinsic::not_intrinsic)
646 return upgradeX86MultiplyAddWords(F, ID, NewFn);
647 }
648 return false; // No other 'x86.avx512.*'.
649 }
650
651 if (Name.consume_front("avx2.")) {
652 if (Name.consume_front("vpdpb")) {
653 // Added in 21.1
654 ID = StringSwitch<Intrinsic::ID>(Name)
655 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
656 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
657 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
658 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
659 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
660 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
661 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
662 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
663 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
664 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
665 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
666 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
667 .Default(Intrinsic::not_intrinsic);
668 if (ID != Intrinsic::not_intrinsic)
669 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
670 } else if (Name.consume_front("vpdpw")) {
671 // Added in 21.1
672 ID = StringSwitch<Intrinsic::ID>(Name)
673 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
674 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
675 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
676 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
677 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
678 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
679 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
680 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
681 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
682 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
683 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
684 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
685 .Default(Intrinsic::not_intrinsic);
686 if (ID != Intrinsic::not_intrinsic)
687 return upgradeX86MultiplyAddWords(F, ID, NewFn);
688 }
689 return false; // No other 'x86.avx2.*'
690 }
691
692 if (Name.consume_front("avx10.")) {
693 if (Name.consume_front("vpdpb")) {
694 // Added in 21.1
695 ID = StringSwitch<Intrinsic::ID>(Name)
696 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
697 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
698 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
699 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
700 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
701 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
702 .Default(Intrinsic::not_intrinsic);
703 if (ID != Intrinsic::not_intrinsic)
704 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
705 } else if (Name.consume_front("vpdpw")) {
706 ID = StringSwitch<Intrinsic::ID>(Name)
707 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
708 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
709 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
710 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
711 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
712 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
713 .Default(Intrinsic::not_intrinsic);
714 if (ID != Intrinsic::not_intrinsic)
715 return upgradeX86MultiplyAddWords(F, ID, NewFn);
716 }
717 return false; // No other 'x86.avx10.*'
718 }
719
720 if (Name.consume_front("avx512bf16.")) {
721 // Added in 9.0
722 ID = StringSwitch<Intrinsic::ID>(Name)
723 .Case("cvtne2ps2bf16.128",
724 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
725 .Case("cvtne2ps2bf16.256",
726 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
727 .Case("cvtne2ps2bf16.512",
728 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
729 .Case("mask.cvtneps2bf16.128",
730 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
731 .Case("cvtneps2bf16.256",
732 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
733 .Case("cvtneps2bf16.512",
734 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
735 .Default(Intrinsic::not_intrinsic);
736 if (ID != Intrinsic::not_intrinsic)
737 return upgradeX86BF16Intrinsic(F, ID, NewFn);
738
739 // Added in 9.0
740 ID = StringSwitch<Intrinsic::ID>(Name)
741 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
742 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
743 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
744 .Default(Intrinsic::not_intrinsic);
745 if (ID != Intrinsic::not_intrinsic)
746 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
747 return false; // No other 'x86.avx512bf16.*'.
748 }
749
750 if (Name.consume_front("xop.")) {
751 Intrinsic::ID ID = Intrinsic::not_intrinsic;
752 if (Name.starts_with("vpermil2")) { // Added in 3.9
753 // Upgrade any XOP PERMIL2 index operand still using a float/double
754 // vector.
755 auto Idx = F->getFunctionType()->getParamType(2);
756 if (Idx->isFPOrFPVectorTy()) {
757 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
758 unsigned EltSize = Idx->getScalarSizeInBits();
759 if (EltSize == 64 && IdxSize == 128)
760 ID = Intrinsic::x86_xop_vpermil2pd;
761 else if (EltSize == 32 && IdxSize == 128)
762 ID = Intrinsic::x86_xop_vpermil2ps;
763 else if (EltSize == 64 && IdxSize == 256)
764 ID = Intrinsic::x86_xop_vpermil2pd_256;
765 else
766 ID = Intrinsic::x86_xop_vpermil2ps_256;
767 }
768 } else if (F->arg_size() == 2)
769 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
770 ID = StringSwitch<Intrinsic::ID>(Name)
771 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
772 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
773 .Default(Intrinsic::not_intrinsic);
774
775 if (ID != Intrinsic::not_intrinsic) {
776 rename(F);
777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
778 return true;
779 }
780 return false; // No other 'x86.xop.*'
781 }
782
783 if (Name == "seh.recoverfp") {
784 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
785 Intrinsic::eh_recoverfp);
786 return true;
787 }
788
789 return false;
790}
791
792// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
793// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
794static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
795 StringRef Name,
796 Function *&NewFn) {
797 if (Name.starts_with("rbit")) {
798 // '(arm|aarch64).rbit'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
801 return true;
802 }
803
804 if (Name == "thread.pointer") {
805 // '(arm|aarch64).thread.pointer'.
806 NewFn = Intrinsic::getOrInsertDeclaration(
807 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
808 return true;
809 }
810
811 bool Neon = Name.consume_front("neon.");
812 if (Neon) {
813 // '(arm|aarch64).neon.*'.
814 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
815 // v16i8 respectively.
816 if (Name.consume_front("bfdot.")) {
817 // (arm|aarch64).neon.bfdot.*'.
818 Intrinsic::ID ID =
819 StringSwitch<Intrinsic::ID>(Name)
820 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
821 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
822 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
823 .Default(Intrinsic::not_intrinsic);
824 if (ID != Intrinsic::not_intrinsic) {
825 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
826 assert((OperandWidth == 64 || OperandWidth == 128) &&
827 "Unexpected operand width");
828 LLVMContext &Ctx = F->getParent()->getContext();
829 std::array<Type *, 2> Tys{
830 {F->getReturnType(),
831 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
832 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
833 return true;
834 }
835 return false; // No other '(arm|aarch64).neon.bfdot.*'.
836 }
837
838 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
839 // anymore and accept v8bf16 instead of v16i8.
840 if (Name.consume_front("bfm")) {
841 // (arm|aarch64).neon.bfm*'.
842 if (Name.consume_back(".v4f32.v16i8")) {
843 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
844 Intrinsic::ID ID =
845 StringSwitch<Intrinsic::ID>(Name)
846 .Case("mla",
847 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
849 .Case("lalb",
850 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
851 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
852 .Case("lalt",
853 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
854 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
855 .Default(Intrinsic::not_intrinsic);
856 if (ID != Intrinsic::not_intrinsic) {
857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
858 return true;
859 }
860 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
861 }
862 return false; // No other '(arm|aarch64).neon.bfm*.
863 }
864 // Continue on to Aarch64 Neon or Arm Neon.
865 }
866 // Continue on to Arm or Aarch64.
867
868 if (IsArm) {
869 // 'arm.*'.
870 if (Neon) {
871 // 'arm.neon.*'.
872 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
873 .StartsWith("vclz.", Intrinsic::ctlz)
874 .StartsWith("vcnt.", Intrinsic::ctpop)
875 .StartsWith("vqadds.", Intrinsic::sadd_sat)
876 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
877 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
878 .StartsWith("vqsubu.", Intrinsic::usub_sat)
879 .StartsWith("vrinta.", Intrinsic::round)
880 .StartsWith("vrintn.", Intrinsic::roundeven)
881 .StartsWith("vrintm.", Intrinsic::floor)
882 .StartsWith("vrintp.", Intrinsic::ceil)
883 .StartsWith("vrintx.", Intrinsic::rint)
884 .StartsWith("vrintz.", Intrinsic::trunc)
885 .Default(Intrinsic::not_intrinsic);
886 if (ID != Intrinsic::not_intrinsic) {
887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
888 F->arg_begin()->getType());
889 return true;
890 }
891
892 if (Name.consume_front("vst")) {
893 // 'arm.neon.vst*'.
894 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
895 SmallVector<StringRef, 2> Groups;
896 if (vstRegex.match(Name, &Groups)) {
897 static const Intrinsic::ID StoreInts[] = {
898 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
899 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
900
901 static const Intrinsic::ID StoreLaneInts[] = {
902 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
903 Intrinsic::arm_neon_vst4lane};
904
905 auto fArgs = F->getFunctionType()->params();
906 Type *Tys[] = {fArgs[0], fArgs[1]};
907 if (Groups[1].size() == 1)
908 NewFn = Intrinsic::getOrInsertDeclaration(
909 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
910 else
911 NewFn = Intrinsic::getOrInsertDeclaration(
912 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
913 return true;
914 }
915 return false; // No other 'arm.neon.vst*'.
916 }
917
918 return false; // No other 'arm.neon.*'.
919 }
920
921 if (Name.consume_front("mve.")) {
922 // 'arm.mve.*'.
923 if (Name == "vctp64") {
924 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
925 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
926 // the function and deal with it below in UpgradeIntrinsicCall.
927 rename(F);
928 return true;
929 }
930 return false; // Not 'arm.mve.vctp64'.
931 }
932
933 if (Name.starts_with("vrintn.v")) {
934 NewFn = Intrinsic::getOrInsertDeclaration(
935 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
936 return true;
937 }
938
939 // These too are changed to accept a v2i1 instead of the old v4i1.
940 if (Name.consume_back(".v4i1")) {
941 // 'arm.mve.*.v4i1'.
942 if (Name.consume_back(".predicated.v2i64.v4i32"))
943 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
944 return Name == "mull.int" || Name == "vqdmull";
945
946 if (Name.consume_back(".v2i64")) {
947 // 'arm.mve.*.v2i64.v4i1'
948 bool IsGather = Name.consume_front("vldr.gather.");
949 if (IsGather || Name.consume_front("vstr.scatter.")) {
950 if (Name.consume_front("base.")) {
951 // Optional 'wb.' prefix.
952 Name.consume_front("wb.");
953 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
954 // predicated.v2i64.v2i64.v4i1'.
955 return Name == "predicated.v2i64";
956 }
957
958 if (Name.consume_front("offset.predicated."))
959 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
960 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
961
962 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
963 return false;
964 }
965
966 return false; // No other 'arm.mve.*.v2i64.v4i1'.
967 }
968 return false; // No other 'arm.mve.*.v4i1'.
969 }
970 return false; // No other 'arm.mve.*'.
971 }
972
973 if (Name.consume_front("cde.vcx")) {
974 // 'arm.cde.vcx*'.
975 if (Name.consume_back(".predicated.v2i64.v4i1"))
976 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
977 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
978 Name == "3q" || Name == "3qa";
979
980 return false; // No other 'arm.cde.vcx*'.
981 }
982 } else {
983 // 'aarch64.*'.
984 if (Neon) {
985 // 'aarch64.neon.*'.
986 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
987 .StartsWith("frintn", Intrinsic::roundeven)
988 .StartsWith("rbit", Intrinsic::bitreverse)
989 .Default(Intrinsic::not_intrinsic);
990 if (ID != Intrinsic::not_intrinsic) {
991 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
992 F->arg_begin()->getType());
993 return true;
994 }
995
996 Intrinsic::ID MinMaxID =
997 StringSwitch<Intrinsic::ID>(Name.split('.').first)
998 .Case("smax", Intrinsic::smax)
999 .Case("smin", Intrinsic::smin)
1000 .Case("umax", Intrinsic::umax)
1001 .Case("umin", Intrinsic::umin)
1002 .Default(Intrinsic::not_intrinsic);
1003 if (MinMaxID != Intrinsic::not_intrinsic) {
1004 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1005 return false; // Invalid IR.
1006 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1007 F->getReturnType());
1008 return true;
1009 }
1010
1011 if (Name.starts_with("addp")) {
1012 // 'aarch64.neon.addp*'.
1013 if (F->arg_size() != 2)
1014 return false; // Invalid IR.
1015 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1016 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1017 NewFn = Intrinsic::getOrInsertDeclaration(
1018 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1019 return true;
1020 }
1021 }
1022
1023 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1024 if (Name.starts_with("bfcvt")) {
1025 NewFn = nullptr;
1026 return true;
1027 }
1028
1029 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1030 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1031 NewFn = nullptr;
1032 return true;
1033 }
1034
1035 return false; // No other 'aarch64.neon.*'.
1036 }
1037 if (Name.consume_front("sve.")) {
1038 // 'aarch64.sve.*'.
1039 if (Name.consume_front("bf")) {
1040 if (Name == "mmla") {
1041 Type *Tys[] = {F->getReturnType(),
1042 std::next(F->arg_begin())->getType()};
1043 NewFn = Intrinsic::getOrInsertDeclaration(
1044 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1045 return true;
1046 }
1047 if (Name.consume_back(".lane")) {
1048 // 'aarch64.sve.bf*.lane'.
1049 Intrinsic::ID ID =
1050 StringSwitch<Intrinsic::ID>(Name)
1051 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1052 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1053 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1054 .Default(Intrinsic::not_intrinsic);
1055 if (ID != Intrinsic::not_intrinsic) {
1056 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1057 return true;
1058 }
1059 return false; // No other 'aarch64.sve.bf*.lane'.
1060 }
1061 return false; // No other 'aarch64.sve.bf*'.
1062 }
1063
1064 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1065 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1066 NewFn = nullptr;
1067 return true;
1068 }
1069
1070 if (Name.consume_front("convert.from.svbool")) {
1071 // 'aarch64.sve.convert.from.svbool'
1072 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1073 if (!TTy || TTy->getName() != "aarch64.svcount")
1074 return false;
1075
1076 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1077 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1078 return true;
1079 }
1080
1081 if (Name.consume_front("convert.to.svbool")) {
1082 // 'aarch64.sve.convert.to.svbool'
1083 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1084 if (!TTy || TTy->getName() != "aarch64.svcount")
1085 return false;
1086
1087 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1088 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1089 return true;
1090 }
1091
1092 if (Name.consume_front("addqv")) {
1093 // 'aarch64.sve.addqv'.
1094 if (!F->getReturnType()->isFPOrFPVectorTy())
1095 return false;
1096
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {F->getReturnType(), Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1101 return true;
1102 }
1103
1104 if (Name.consume_front("ld")) {
1105 // 'aarch64.sve.ld*'.
1106 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1107 if (LdRegex.match(Name)) {
1108 Type *ScalarTy =
1109 cast<VectorType>(F->getReturnType())->getElementType();
1110 ElementCount EC =
1111 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1112 assert(F->arg_size() == 2 &&
1113 "Expected 2 arguments for ld* intrinsic.");
1114 Type *PtrTy = F->getArg(1)->getType();
1115 Type *Ty = VectorType::get(ScalarTy, EC);
1116 static const Intrinsic::ID LoadIDs[] = {
1117 Intrinsic::aarch64_sve_ld2_sret,
1118 Intrinsic::aarch64_sve_ld3_sret,
1119 Intrinsic::aarch64_sve_ld4_sret,
1120 };
1121 NewFn = Intrinsic::getOrInsertDeclaration(
1122 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1123 return true;
1124 }
1125 return false; // No other 'aarch64.sve.ld*'.
1126 }
1127
1128 if (Name.consume_front("tuple.")) {
1129 // 'aarch64.sve.tuple.*'.
1130 if (Name.starts_with("get")) {
1131 // 'aarch64.sve.tuple.get*'.
1132 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1133 NewFn = Intrinsic::getOrInsertDeclaration(
1134 F->getParent(), Intrinsic::vector_extract, Tys);
1135 return true;
1136 }
1137
1138 if (Name.starts_with("set")) {
1139 // 'aarch64.sve.tuple.set*'.
1140 auto Args = F->getFunctionType()->params();
1141 Type *Tys[] = {Args[0], Args[2], Args[1]};
1142 NewFn = Intrinsic::getOrInsertDeclaration(
1143 F->getParent(), Intrinsic::vector_insert, Tys);
1144 return true;
1145 }
1146
1147 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1148 if (CreateTupleRegex.match(Name)) {
1149 // 'aarch64.sve.tuple.create*'.
1150 auto Args = F->getFunctionType()->params();
1151 Type *Tys[] = {F->getReturnType(), Args[1]};
1152 NewFn = Intrinsic::getOrInsertDeclaration(
1153 F->getParent(), Intrinsic::vector_insert, Tys);
1154 return true;
1155 }
1156 return false; // No other 'aarch64.sve.tuple.*'.
1157 }
1158
1159 if (Name.starts_with("rev.nxv")) {
1160 // 'aarch64.sve.rev.<Ty>'
1161 NewFn = Intrinsic::getOrInsertDeclaration(
1162 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1163 return true;
1164 }
1165
1166 return false; // No other 'aarch64.sve.*'.
1167 }
1168 if (Name.consume_front("sme.")) {
1169 // 'aarch64.sme.*'.
1170 if (Name.consume_front("ftmopa.")) {
1171 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1172 // intrinsics to model their FPMR dependency.
1173 Intrinsic::ID ID =
1174 StringSwitch<Intrinsic::ID>(Name)
1175 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1176 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1177 .Default(Intrinsic::not_intrinsic);
1178 if (ID != Intrinsic::not_intrinsic) {
1179 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1180 return true;
1181 }
1182 return false; // No other 'aarch64.sme.ftmopa.*'.
1183 }
1184
1185 return false; // No other 'aarch64.sme.*'.
1186 }
1187 }
1188 return false; // No other 'arm.*', 'aarch64.*'.
1189}
1190
1191// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1192// declarations requiring an auto-upgrade. The same set applies to the
1193// cluster (g2s) and CTA (g2s_cta) variants.
1194#define NVVM_TMA_G2S_MODES(M) \
1195 M(tile_1d, "tile.1d") \
1196 M(tile_2d, "tile.2d") \
1197 M(tile_3d, "tile.3d") \
1198 M(tile_4d, "tile.4d") \
1199 M(tile_5d, "tile.5d") \
1200 M(tile_gather4_2d, "tile.gather4.2d") \
1201 M(im2col_3d, "im2col.3d") \
1202 M(im2col_4d, "im2col.4d") \
1203 M(im2col_5d, "im2col.5d") \
1204 M(im2col_w_3d, "im2col.w.3d") \
1205 M(im2col_w_4d, "im2col.w.4d") \
1206 M(im2col_w_5d, "im2col.w.5d") \
1207 M(im2col_w_128_3d, "im2col.w.128.3d") \
1208 M(im2col_w_128_4d, "im2col.w.128.4d") \
1209 M(im2col_w_128_5d, "im2col.w.128.5d")
1210
1211// Two legacy tails are:
1212//
1213// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1215//
1216// The current tail appends a trailing i32 %flag_valid_pattern, so both
1217// legacy tails are recognized by an i1 at parameter N-2.
1218static Intrinsic::ID
1219shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1220 SmallVectorImpl<Type *> &OvlTys) {
1221 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1222 return Intrinsic::not_intrinsic;
1223
1224#define G2S_ID(ID_SUFFIX, NAME) \
1225 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1226 // clang-format off
1227 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1228 NVVM_TMA_G2S_MODES(G2S_ID)
1229 .Default(Intrinsic::not_intrinsic);
1230#undef G2S_ID
1231 // clang-format on
1232 if (ID == Intrinsic::not_intrinsic)
1233 return ID;
1234
1235 size_t NumParams = F->getFunctionType()->getNumParams();
1236
1237 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1238 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1239 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1240 return Intrinsic::not_intrinsic;
1241
1242 // The multicast mask is the parameter immediately before the i64
1243 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1244 ArrayRef<Type *> Params = F->getFunctionType()->params();
1245 size_t MaskIdx =
1246 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1247 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1248 "expected the i64 cache-hint after the multicast mask");
1249 Type *MaskTy = Params[MaskIdx];
1250 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1251 OvlTys.push_back(MaskTy);
1252
1253 return ID;
1254}
1255
1256// The legacy tail is:
1257//
1258// arg1, arg2, .. i64 %ch, i1 %flag_ch
1259//
1260// The current tail appends a trailing i32 %flag_valid_pattern, so the
1261// legacy tail is recognized by an i1 at parameter N-1.
1262static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1263 StringRef Name) {
1264 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1265 return Intrinsic::not_intrinsic;
1266
1267#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1268 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1269 // clang-format off
1270 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1271 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1272 .Default(Intrinsic::not_intrinsic);
1273#undef G2S_CTA_ID
1274 // clang-format on
1275 if (ID == Intrinsic::not_intrinsic)
1276 return ID;
1277
1278 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1279 // with i32 %flag_valid_pattern, for which N-1 is i32.
1280 if (!F->getFunctionType()
1281 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1282 ->isIntegerTy(1))
1283 return Intrinsic::not_intrinsic;
1284
1285 return ID;
1286}
1287// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1288//
1289// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1290//
1291// The current intrinsic is overloaded on the multicast-mask type and takes a
1292// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1293// parameter N-1.
1294static Intrinsic::ID
1295shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1296 SmallVectorImpl<Type *> &OvlTys) {
1297 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1298 return Intrinsic::not_intrinsic;
1299
1300 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1301 // i32 %flag_valid_pattern, for which N-1 is i32.
1302 size_t NumParams = F->getFunctionType()->getNumParams();
1303 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1304 return Intrinsic::not_intrinsic;
1305
1306 // The multicast mask is parameter 4; legacy IR only uses i16.
1307 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1308 if (!MaskTy->isIntegerTy(16))
1309 return Intrinsic::not_intrinsic;
1310 OvlTys.push_back(MaskTy);
1311
1312 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1313}
1314
1315// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1316//
1317// ..., i64 %ch, i1 %flag_ch
1318//
1319// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1320// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1321// recognized by an i1 at parameter N-1, whereas the current tail ends
1322// with an i32.
1323static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1324 StringRef Name) {
1325 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1326 return Intrinsic::not_intrinsic;
1327
1328 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1329 // i32 %flag_valid_pattern, for which N-1 is i32.
1330 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1331 return Intrinsic::not_intrinsic;
1332
1333 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1334}
1335
1336// The legacy TMA reduction intrinsics encode the reduction operator in their
1337// name, while the current ones take it as an immediate argument. Map the
1338// operator part of a legacy name to the corresponding immediate value.
1339static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1340 return StringSwitch<std::optional<unsigned>>(Name)
1341 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1342 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1343 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1344 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1345 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1346 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1347 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1348 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1349 .Default(std::nullopt);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1353 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1354 return Intrinsic::not_intrinsic;
1355
1356 auto [RedOpName, ShapeName] = Name.split('.');
1357 if (!getNVPTXTMAReductionOp(RedOpName))
1358 return Intrinsic::not_intrinsic;
1359
1360 return StringSwitch<Intrinsic::ID>(ShapeName)
1361 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1362 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1363 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1364 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1365 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1366 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1367 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1368 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1369 .Default(Intrinsic::not_intrinsic);
1370}
1371
1372static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1373 StringRef Name) {
1374 if (Name.consume_front("mapa.shared.cluster"))
1375 if (F->getReturnType()->getPointerAddressSpace() ==
1376 NVPTXAS::ADDRESS_SPACE_SHARED)
1377 return Intrinsic::nvvm_mapa_shared_cluster;
1378
1379 if (Name.consume_front("cp.async.bulk.")) {
1380 Intrinsic::ID ID =
1381 StringSwitch<Intrinsic::ID>(Name)
1382 .Case("shared.cta.to.cluster",
1383 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1384 .Default(Intrinsic::not_intrinsic);
1385
1386 if (ID != Intrinsic::not_intrinsic)
1387 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1388 NVPTXAS::ADDRESS_SPACE_SHARED)
1389 return ID;
1390 }
1391
1392 return Intrinsic::not_intrinsic;
1393}
1394
1395static Intrinsic::ID
1396shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1397 if (!Name.consume_front("tcgen05.commit."))
1398 return Intrinsic::not_intrinsic;
1399
1400 if (Name.consume_front("shared."))
1401 return StringSwitch<Intrinsic::ID>(Name)
1402 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1403 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1404 .Default(Intrinsic::not_intrinsic);
1405
1406 if (Name.consume_front("mc.shared.")) {
1407 // Only upgrade older i16 mc variants.
1408 if (!F->getArg(1)->getType()->isIntegerTy(16))
1409 return Intrinsic::not_intrinsic;
1410
1411 return StringSwitch<Intrinsic::ID>(Name)
1412 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1413 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1414 .Default(Intrinsic::not_intrinsic);
1415 }
1416
1417 return Intrinsic::not_intrinsic;
1418}
1419
1420static Intrinsic::ID
1421shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1422 if (F->arg_size() != 2)
1423 return Intrinsic::not_intrinsic;
1424
1425 if (Name.consume_front("tcgen05.alloc.shared.") ||
1426 Name.consume_front("tcgen05.alloc."))
1427 return StringSwitch<Intrinsic::ID>(Name)
1428 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1429 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1430 .Default(Intrinsic::not_intrinsic);
1431
1432 if (Name.consume_front("tcgen05.dealloc."))
1433 return StringSwitch<Intrinsic::ID>(Name)
1434 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1435 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1436 .Default(Intrinsic::not_intrinsic);
1437
1438 return Intrinsic::not_intrinsic;
1439}
1440
1441static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1442 if (Name.consume_front("fmax."))
1443 return StringSwitch<Intrinsic::ID>(Name)
1444 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1445 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1446 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1447 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1448 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1449 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1450 .Case("ftz.nan.xorsign.abs.bf16",
1451 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1452 .Case("ftz.nan.xorsign.abs.bf16x2",
1453 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1454 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1455 .Case("ftz.xorsign.abs.bf16x2",
1456 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1457 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1458 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1459 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1460 .Case("nan.xorsign.abs.bf16x2",
1461 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1462 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1463 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1464 .Default(Intrinsic::not_intrinsic);
1465
1466 if (Name.consume_front("fmin."))
1467 return StringSwitch<Intrinsic::ID>(Name)
1468 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1469 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1470 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1471 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1472 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1473 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1474 .Case("ftz.nan.xorsign.abs.bf16",
1475 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1476 .Case("ftz.nan.xorsign.abs.bf16x2",
1477 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1478 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1479 .Case("ftz.xorsign.abs.bf16x2",
1480 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1481 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1482 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1483 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1484 .Case("nan.xorsign.abs.bf16x2",
1485 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1486 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1487 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1488 .Default(Intrinsic::not_intrinsic);
1489
1490 if (Name.consume_front("neg."))
1491 return StringSwitch<Intrinsic::ID>(Name)
1492 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1493 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1494 .Default(Intrinsic::not_intrinsic);
1495
1496 return Intrinsic::not_intrinsic;
1497}
1498
1499static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1500 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1501 FunctionType *OldFnTy = F->getFunctionType();
1502 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1503 return OldTy->getScalarType()->isIntegerTy() &&
1504 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1505 };
1506
1507 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1508 return false;
1509
1510 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1511 return false;
1512
1513 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1514 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1515 return false;
1516
1517 return true;
1518}
1519
1520// Overloaded fadd/fmul/fma intrinsic IDs, indexed by
1521// [`.ftz`][none|`.sat`|`.relu`].
1522static constexpr Intrinsic::ID NVVMFAddIIDs[2][3] = {
1523 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
1524 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1525 Intrinsic::not_intrinsic}};
1526static constexpr Intrinsic::ID NVVMFMulIIDs[2][3] = {
1527 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
1528 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1529 Intrinsic::not_intrinsic}};
1530static constexpr Intrinsic::ID NVVMFMAIIDs[2][3] = {
1531 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1532 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1533 Intrinsic::nvvm_fma_ftz_relu}};
1534
1535// Returns true if the legacy nvvm.{add,mul,fma}.<rnd>{.ftz}{.sat|.relu}.<Type>
1536// intrinsic with these modifiers existed.
1537static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID (&IIDs)[2][3],
1538 StringRef Type, bool IsRN,
1539 bool IsFTZ, bool IsSat,
1540 bool IsRelu) {
1541 const bool IsFMA = &IIDs == &NVVMFMAIIDs;
1542 const bool IsFMul = &IIDs == &NVVMFMulIIDs;
1543 return StringSwitch<bool>(Type)
1544 .Case("f", !IsRelu && !(IsSat && IsFMul))
1545 .Case("d", !IsFTZ && !IsSat && !IsRelu)
1546 .Case("f16", IsRN && (IsFMA || IsSat))
1547 .Case("v2f16", IsRN && !IsFMA && IsSat)
1548 .Case("f16x2", IsRN && IsFMA)
1549 .Cases({"bf16", "bf16x2"}, IsRN && IsFMA && !IsFTZ && !IsSat)
1550 .Default(false);
1551}
1552
1553static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1554getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][3]) {
1555 auto [Modifiers, Type] = Name.rsplit('.');
1556
1557 std::optional<llvm::RoundingMode> RoundingMode =
1558 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1559 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1560 .Case("rz", llvm::RoundingMode::TowardZero)
1561 .Case("rm", llvm::RoundingMode::TowardNegative)
1562 .Case("rp", llvm::RoundingMode::TowardPositive)
1563 .Default(std::nullopt);
1564 if (!RoundingMode)
1565 return std::nullopt;
1566
1567 StringRef Rest = Modifiers.drop_front(2);
1568 const bool IsFTZ = Rest.consume_front(".ftz");
1569 const bool IsSat = Rest.consume_front(".sat");
1570 const bool IsRelu = !IsSat && Rest.consume_front(".relu");
1571 if (!Rest.empty())
1572 return std::nullopt;
1573
1574 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1575 if (IID == Intrinsic::not_intrinsic)
1576 return std::nullopt;
1577
1578 // Reject combinations that never existed as legacy intrinsics.
1579 const bool IsRN = *RoundingMode == llvm::RoundingMode::NearestTiesToEven;
1580 if (!isValidLegacyNVVMFPArithIntrinsic(IIDs, Type, IsRN, IsFTZ, IsSat,
1581 IsRelu))
1582 return std::nullopt;
1583
1584 return std::make_pair(IID, *RoundingMode);
1585}
1586
1587static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1588 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1589 return Intrinsic::not_intrinsic;
1590
1591 return Intrinsic::nvvm_mbarrier_init;
1592}
1593
1594static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1595 return Name.consume_front("local") || Name.consume_front("shared") ||
1596 Name.consume_front("global") || Name.consume_front("constant") ||
1597 Name.consume_front("param");
1598}
1599
1600static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1601 if (!Name.consume_front("vp."))
1602 return 0;
1603 return StringSwitch<unsigned>(Name)
1604 .StartsWith("select", Instruction::Select)
1605 .StartsWith("add", Instruction::Add)
1606 .StartsWith("sub", Instruction::Sub)
1607 .StartsWith("mul", Instruction::Mul)
1608 .StartsWith("ashr", Instruction::AShr)
1609 .StartsWith("lshr", Instruction::LShr)
1610 .StartsWith("shl", Instruction::Shl)
1611 .StartsWith("or", Instruction::Or)
1612 .StartsWith("and", Instruction::And)
1613 .StartsWith("xor", Instruction::Xor)
1614 .StartsWith("fadd", Instruction::FAdd)
1615 .StartsWith("fsub", Instruction::FSub)
1616 .StartsWith("fmuladd", 0)
1617 .StartsWith("fmul", Instruction::FMul)
1618 .StartsWith("fdiv", Instruction::FDiv)
1619 .StartsWith("frem", Instruction::FRem)
1620 .StartsWith("fneg", Instruction::FNeg)
1621 .StartsWith("trunc", Instruction::Trunc)
1622 .StartsWith("zext", Instruction::ZExt)
1623 .StartsWith("sext", Instruction::SExt)
1624 .StartsWith("fptrunc", Instruction::FPTrunc)
1625 .StartsWith("fpext", Instruction::FPExt)
1626 .StartsWith("fptoui", Instruction::FPToUI)
1627 .StartsWith("fptosi", Instruction::FPToSI)
1628 .StartsWith("uitofp", Instruction::UIToFP)
1629 .StartsWith("sitofp", Instruction::SIToFP)
1630 .StartsWith("ptrtoint", Instruction::PtrToInt)
1631 .StartsWith("inttoptr", Instruction::IntToPtr)
1632 .StartsWith("icmp", Instruction::ICmp)
1633 .StartsWith("fcmp", Instruction::FCmp)
1634 .Default(0);
1635}
1636
1637static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1638 if (!Name.consume_front("vp."))
1639 return 0;
1640 return StringSwitch<Intrinsic::ID>(Name)
1641 .StartsWith("abs", Intrinsic::abs)
1642 .StartsWith("smax", Intrinsic::smax)
1643 .StartsWith("smin", Intrinsic::smin)
1644 .StartsWith("umax", Intrinsic::umax)
1645 .StartsWith("umin", Intrinsic::umin)
1646 .StartsWith("copysign", Intrinsic::copysign)
1647 .StartsWith("minnum", Intrinsic::minnum)
1648 .StartsWith("maxnum", Intrinsic::maxnum)
1649 .StartsWith("minimum", Intrinsic::minimum)
1650 .StartsWith("maximum", Intrinsic::maximum)
1651 .StartsWith("fabs", Intrinsic::fabs)
1652 .StartsWith("sqrt", Intrinsic::sqrt)
1653 .StartsWith("fma", Intrinsic::fma)
1654 .StartsWith("fmuladd", Intrinsic::fmuladd)
1655 .StartsWith("ceil", Intrinsic::ceil)
1656 .StartsWith("floor", Intrinsic::floor)
1657 .StartsWith("rint", Intrinsic::rint)
1658 .StartsWith("nearbyint", Intrinsic::nearbyint)
1659 .StartsWith("roundeven", Intrinsic::roundeven)
1660 .StartsWith("roundtozero", Intrinsic::trunc)
1661 .StartsWith("round", Intrinsic::round)
1662 .StartsWith("lrint", Intrinsic::lrint)
1663 .StartsWith("llrint", Intrinsic::llrint)
1664 .StartsWith("bitreverse", Intrinsic::bitreverse)
1665 .StartsWith("bswap", Intrinsic::bswap)
1666 .StartsWith("ctpop", Intrinsic::ctpop)
1667 .StartsWith("ctlz", Intrinsic::ctlz)
1668 .StartsWith("cttz.elts", 0)
1669 .StartsWith("cttz", Intrinsic::cttz)
1670 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1671 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1672 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1673 .StartsWith("usub.sat", Intrinsic::usub_sat)
1674 .StartsWith("fshl", Intrinsic::fshl)
1675 .StartsWith("fshr", Intrinsic::fshr)
1676 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1677 .Default(0);
1678}
1679
1683
1684static bool convertIntrinsicValidType(StringRef Name,
1685 const FunctionType *FuncTy) {
1686 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1687 if (Name.starts_with("to.fp16")) {
1688 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1689 HalfTy) &&
1690 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1691 FuncTy->getReturnType());
1692 }
1693
1694 if (Name.starts_with("from.fp16")) {
1695 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1696 HalfTy) &&
1697 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1698 FuncTy->getReturnType());
1699 }
1700
1701 return false;
1702}
1703
1704static unsigned
1705getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1706 SmallVectorImpl<Type *> &OverloadTys) {
1707 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1708 if (Defaults.empty())
1709 return 0;
1710
1711 unsigned FullArgCount = FirstDefault + Defaults.size();
1712
1713 // Only trailing default arguments can be missing.
1714 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1715 return 0;
1716
1717 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1718 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1719 NumMissingTrailingParams))
1720 return 0;
1721
1722 return FullArgCount;
1723}
1724
1725static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1726 Intrinsic::ID IID = F->getIntrinsicID();
1727 SmallVector<Type *, 4> OverloadTys;
1728
1729 unsigned FullArgCount =
1730 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1731 if (FullArgCount == 0)
1732 return false;
1733
1734 rename(F);
1735 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1736 assert(NewFn->arg_size() == FullArgCount &&
1737 "total number of default args does not match intrinsic signature");
1738 return true;
1739}
1740
1741static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1742 bool CanUpgradeDebugIntrinsicsToRecords) {
1743 assert(F && "Illegal to upgrade a non-existent Function.");
1744
1745 StringRef Name = F->getName();
1746
1747 // Quickly eliminate it, if it's not a candidate.
1748 if (!Name.consume_front("llvm.") || Name.empty())
1749 return false;
1750
1751 switch (Name[0]) {
1752 default: break;
1753 case 'a': {
1754 bool IsArm = Name.consume_front("arm.");
1755 if (IsArm || Name.consume_front("aarch64.")) {
1756 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1757 return true;
1758 break;
1759 }
1760
1761 if (Name.consume_front("amdgcn.")) {
1762 if (Name == "alignbit") {
1763 // Target specific intrinsic became redundant
1764 NewFn = Intrinsic::getOrInsertDeclaration(
1765 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1766 return true;
1767 }
1768
1769 if (Name.consume_front("atomic.")) {
1770 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1771 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1772 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1773 // and usub_sat so there's no new declaration.
1774 NewFn = nullptr;
1775 return true;
1776 }
1777 break; // No other 'amdgcn.atomic.*'
1778 }
1779
1780 if (Name.starts_with("addrspacecast.nonnull")) {
1781 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1782 // so there's no new declaration.
1783 NewFn = nullptr;
1784 return true;
1785 }
1786
1787 switch (F->getIntrinsicID()) {
1788 default:
1789 break;
1790 // Legacy wmma iu intrinsics without the optional clamp operand.
1791 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1792 if (F->arg_size() == 7) {
1793 NewFn = nullptr;
1794 return true;
1795 }
1796 break;
1797 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1798 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1799 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1800 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1801 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1802 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1803 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1804 if (F->arg_size() == 8) {
1805 NewFn = nullptr;
1806 return true;
1807 }
1808 break;
1809 }
1810
1811 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1812 Name.consume_front("flat.atomic.")) {
1813 if (Name.starts_with("fadd") ||
1814 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1815 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1816 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1817 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1818 // declaration.
1819 NewFn = nullptr;
1820 return true;
1821 }
1822 }
1823
1824 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1825 NewFn = nullptr;
1826 return true;
1827 }
1828
1829 if (Name.starts_with("ldexp.")) {
1830 // Target specific intrinsic became redundant
1831 NewFn = Intrinsic::getOrInsertDeclaration(
1832 F->getParent(), Intrinsic::ldexp,
1833 {F->getReturnType(), F->getArg(1)->getType()});
1834 return true;
1835 }
1836 break; // No other 'amdgcn.*'
1837 }
1838
1839 break;
1840 }
1841 case 'c': {
1842 if (F->arg_size() == 1) {
1843 if (Name.consume_front("convert.")) {
1844 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1845 NewFn = nullptr;
1846 return true;
1847 }
1848 }
1849
1850 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1851 .StartsWith("ctlz.", Intrinsic::ctlz)
1852 .StartsWith("cttz.", Intrinsic::cttz)
1853 .Default(Intrinsic::not_intrinsic);
1854 if (ID != Intrinsic::not_intrinsic) {
1855 rename(F);
1856 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1857 F->arg_begin()->getType());
1858 return true;
1859 }
1860 }
1861
1862 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1863 if (Name == "coro.end" &&
1864 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1865 CoroEndID = Intrinsic::coro_end;
1866 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1867 CoroEndID = Intrinsic::coro_end_async;
1868
1869 if (CoroEndID != Intrinsic::not_intrinsic) {
1870 rename(F);
1871 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1872 return true;
1873 }
1874
1875 break;
1876 }
1877 case 'd':
1878 if (Name.consume_front("dbg.")) {
1879 // Mark debug intrinsics for upgrade to new debug format.
1880 if (CanUpgradeDebugIntrinsicsToRecords) {
1881 if (Name == "addr" || Name == "value" || Name == "assign" ||
1882 Name == "declare" || Name == "label") {
1883 // There's no function to replace these with.
1884 NewFn = nullptr;
1885 // But we do want these to get upgraded.
1886 return true;
1887 }
1888 }
1889 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1890 // converted to DbgVariableRecords later.
1891 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1892 rename(F);
1893 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1894 Intrinsic::dbg_value);
1895 return true;
1896 }
1897 break; // No other 'dbg.*'.
1898 }
1899 break;
1900 case 'e':
1901 if (Name.consume_front("experimental.vector.")) {
1902 Intrinsic::ID ID =
1903 StringSwitch<Intrinsic::ID>(Name)
1904 // Skip over extract.last.active, otherwise it will be 'upgraded'
1905 // to a regular vector extract which is a different operation.
1906 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1907 .StartsWith("extract.", Intrinsic::vector_extract)
1908 .StartsWith("insert.", Intrinsic::vector_insert)
1909 .StartsWith("reverse.", Intrinsic::vector_reverse)
1910 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1911 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1912 .StartsWith("partial.reduce.add",
1913 Intrinsic::vector_partial_reduce_add)
1914 .Default(Intrinsic::not_intrinsic);
1915 if (ID != Intrinsic::not_intrinsic) {
1916 const auto *FT = F->getFunctionType();
1917 SmallVector<Type *, 2> Tys;
1918 if (ID == Intrinsic::vector_extract ||
1919 ID == Intrinsic::vector_interleave2)
1920 // Extracting overloads the return type.
1921 Tys.push_back(FT->getReturnType());
1922 if (ID != Intrinsic::vector_interleave2)
1923 Tys.push_back(FT->getParamType(0));
1924 if (ID == Intrinsic::vector_insert ||
1925 ID == Intrinsic::vector_partial_reduce_add)
1926 // Inserting overloads the inserted type.
1927 Tys.push_back(FT->getParamType(1));
1928 rename(F);
1929 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1930 return true;
1931 }
1932
1933 if (Name.consume_front("reduce.")) {
1934 SmallVector<StringRef, 2> Groups;
1935 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1936 if (R.match(Name, &Groups))
1937 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1938 .Case("add", Intrinsic::vector_reduce_add)
1939 .Case("mul", Intrinsic::vector_reduce_mul)
1940 .Case("and", Intrinsic::vector_reduce_and)
1941 .Case("or", Intrinsic::vector_reduce_or)
1942 .Case("xor", Intrinsic::vector_reduce_xor)
1943 .Case("smax", Intrinsic::vector_reduce_smax)
1944 .Case("smin", Intrinsic::vector_reduce_smin)
1945 .Case("umax", Intrinsic::vector_reduce_umax)
1946 .Case("umin", Intrinsic::vector_reduce_umin)
1947 .Case("fmax", Intrinsic::vector_reduce_fmax)
1948 .Case("fmin", Intrinsic::vector_reduce_fmin)
1949 .Default(Intrinsic::not_intrinsic);
1950
1951 bool V2 = false;
1952 if (ID == Intrinsic::not_intrinsic) {
1953 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1954 Groups.clear();
1955 V2 = true;
1956 if (R2.match(Name, &Groups))
1957 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1958 .Case("fadd", Intrinsic::vector_reduce_fadd)
1959 .Case("fmul", Intrinsic::vector_reduce_fmul)
1960 .Default(Intrinsic::not_intrinsic);
1961 }
1962 if (ID != Intrinsic::not_intrinsic) {
1963 rename(F);
1964 auto Args = F->getFunctionType()->params();
1965 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1966 {Args[V2 ? 1 : 0]});
1967 return true;
1968 }
1969 break; // No other 'expermental.vector.reduce.*'.
1970 }
1971
1972 if (Name.consume_front("splice"))
1973 return true;
1974 break; // No other 'experimental.vector.*'.
1975 }
1976 if (Name.consume_front("experimental.stepvector.")) {
1977 Intrinsic::ID ID = Intrinsic::stepvector;
1978 rename(F);
1979 NewFn = Intrinsic::getOrInsertDeclaration(
1980 F->getParent(), ID, F->getFunctionType()->getReturnType());
1981 return true;
1982 }
1983 break; // No other 'e*'.
1984 case 'f':
1985 if (Name.starts_with("flt.rounds")) {
1986 rename(F);
1987 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1988 Intrinsic::get_rounding);
1989 return true;
1990 }
1991 break;
1992 case 'i':
1993 if (Name.starts_with("invariant.group.barrier")) {
1994 // Rename invariant.group.barrier to launder.invariant.group
1995 auto Args = F->getFunctionType()->params();
1996 Type* ObjectPtr[1] = {Args[0]};
1997 rename(F);
1998 NewFn = Intrinsic::getOrInsertDeclaration(
1999 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
2000 return true;
2001 }
2002 break;
2003 case 'l': {
2004 bool IsLifetimeStart = Name.consume_front("lifetime.start");
2005 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
2006 if (IsLifetimeStart || IsLifetimeEnd) {
2007 if (F->arg_size() == 2) {
2008 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2009 : Intrinsic::lifetime_end;
2010 rename(F);
2011 // Old 2 argument form of these intrinsics have [Size, Ptr] as
2012 // arguments. Use the Ptr argument to create new declaration.
2013 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2014 F->getArg(1)->getType());
2015 return true;
2016 } else if (F->arg_size() == 1 && Name == ".i64") {
2017 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
2018 // Autoupgrade prior to
2019 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
2020 // intrinsic with no expected calls. To allow auto-upgrade process to
2021 // delete such invalid intrinsic declaration, set NewFn = nullptr
2022 // and return true here. If there are actual calls to this intrinsic
2023 // (which is not expected), they will be deleted in
2024 // UpgradeIntrinsicCall.
2025 NewFn = nullptr;
2026 return true;
2027 }
2028 }
2029 break;
2030 }
2031 case 'm': {
2032 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2033 // alignment parameter to embedding the alignment as an attribute of
2034 // the pointer args.
2035 if (unsigned ID = StringSwitch<unsigned>(Name)
2036 .StartsWith("memcpy.", Intrinsic::memcpy)
2037 .StartsWith("memmove.", Intrinsic::memmove)
2038 .Default(0)) {
2039 if (F->arg_size() == 5) {
2040 rename(F);
2041 // Get the types of dest, src, and len
2042 ArrayRef<Type *> ParamTypes =
2043 F->getFunctionType()->params().slice(0, 3);
2044 NewFn =
2045 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2046 return true;
2047 }
2048 }
2049 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2050 rename(F);
2051 // Get the types of dest, and len
2052 const auto *FT = F->getFunctionType();
2053 Type *ParamTypes[2] = {
2054 FT->getParamType(0), // Dest
2055 FT->getParamType(2) // len
2056 };
2057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2058 Intrinsic::memset, ParamTypes);
2059 return true;
2060 }
2061
2062 unsigned MaskedID =
2063 StringSwitch<unsigned>(Name)
2064 .StartsWith("masked.load", Intrinsic::masked_load)
2065 .StartsWith("masked.gather", Intrinsic::masked_gather)
2066 .StartsWith("masked.store", Intrinsic::masked_store)
2067 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2068 .Default(0);
2069 if (MaskedID && F->arg_size() == 4) {
2070 rename(F);
2071 if (MaskedID == Intrinsic::masked_load ||
2072 MaskedID == Intrinsic::masked_gather) {
2073 NewFn = Intrinsic::getOrInsertDeclaration(
2074 F->getParent(), MaskedID,
2075 {F->getReturnType(), F->getArg(0)->getType()});
2076 return true;
2077 }
2078 NewFn = Intrinsic::getOrInsertDeclaration(
2079 F->getParent(), MaskedID,
2080 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2081 return true;
2082 }
2083 break;
2084 }
2085 case 'n': {
2086 if (Name.consume_front("nvvm.")) {
2087 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2088 if (F->arg_size() == 1) {
2089 Intrinsic::ID IID =
2090 StringSwitch<Intrinsic::ID>(Name)
2091 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2092 .Case("clz.i", Intrinsic::ctlz)
2093 .Case("popc.i", Intrinsic::ctpop)
2094 .Default(Intrinsic::not_intrinsic);
2095 if (IID != Intrinsic::not_intrinsic) {
2096 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2097 {F->getReturnType()});
2098 return true;
2099 }
2100 } else if (F->arg_size() == 2) {
2101 Intrinsic::ID IID =
2102 StringSwitch<Intrinsic::ID>(Name)
2103 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2104 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2105 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2106 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2107 .Cases({"mulhi.s", "mulhi.i", "mulhi.ll"}, Intrinsic::smulh)
2108 .Cases({"mulhi.us", "mulhi.ui", "mulhi.ull"}, Intrinsic::umulh)
2109 .Default(Intrinsic::not_intrinsic);
2110 if (IID != Intrinsic::not_intrinsic) {
2111 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2112 {F->getReturnType()});
2113 return true;
2114 }
2115 }
2116
2117 // Check for nvvm intrinsics that need a return type adjustment.
2118 {
2119 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2120 if (IID != Intrinsic::not_intrinsic &&
2121 isLegacyNVPTXBF16IntSignature(F, IID)) {
2122 NewFn = nullptr;
2123 return true;
2124 }
2125 }
2126
2127 // Upgrade Distributed Shared Memory Intrinsics
2128 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2129 if (IID != Intrinsic::not_intrinsic) {
2130 rename(F);
2131 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2132 return true;
2133 }
2134
2135 // Upgrade TMA reduction intrinsics
2136 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2137 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2138 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2139 if (IID != Intrinsic::not_intrinsic) {
2140 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2141 return true;
2142 }
2143
2144 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2145 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2146 if (IID != Intrinsic::not_intrinsic) {
2147 rename(F);
2148 NewFn = Intrinsic::getOrInsertDeclaration(
2149 F->getParent(), IID, F->getReturnType(),
2150 F->getFunctionType()->params());
2151 return true;
2152 }
2153
2154 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2155 // tcgen05.alloc shared variants to anyptr intrinsics.
2156 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2157 if (IID != Intrinsic::not_intrinsic) {
2158 rename(F);
2159 if (Intrinsic::isOverloaded(IID))
2160 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2161 {F->getArg(0)->getType()});
2162 else
2163 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2164 return true;
2165 }
2166
2167 // Upgrade TMA copy G2S CTA intrinsics.
2168 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2169 if (IID != Intrinsic::not_intrinsic) {
2170 rename(F);
2171 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2172 return true;
2173 }
2174
2175 // Upgrade TMA copy G2S (cluster) intrinsics.
2176 SmallVector<Type *, 1> OvlTys;
2177 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2178 if (IID != Intrinsic::not_intrinsic) {
2179 rename(F);
2180 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2181 return true;
2182 }
2183
2184 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2185 // (multicast-mask overloading + trailing flag_valid_pattern).
2186 SmallVector<Type *, 1> BulkG2SOvlTys;
2187 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2188 if (IID != Intrinsic::not_intrinsic) {
2189 rename(F);
2190 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2191 BulkG2SOvlTys);
2192 return true;
2193 }
2194
2195 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2196 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2197 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2198 if (IID != Intrinsic::not_intrinsic) {
2199 rename(F);
2200 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2201 return true;
2202 }
2203
2204 // Upgrade mbarrier.init intrinsics missing the layout operand.
2205 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2206 if (IID != Intrinsic::not_intrinsic) {
2207 rename(F);
2208 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2209 F->getArg(0)->getType());
2210 return true;
2211 }
2212
2213 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2214 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2215 //
2216 // TODO: We could add lohi.i2d.
2217 bool Expand = false;
2218 if (Name.consume_front("abs."))
2219 // nvvm.abs.{i,ii}
2220 Expand =
2221 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2222 else if (Name.consume_front("fabs."))
2223 // nvvm.fabs.{f,ftz.f,d}
2224 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2225 else if (Name.consume_front("add."))
2226 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2227 Expand = getNVVMFPArithUpgrade(Name, NVVMFAddIIDs).has_value();
2228 else if (Name.consume_front("mul."))
2229 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2230 Expand = getNVVMFPArithUpgrade(Name, NVVMFMulIIDs).has_value();
2231 else if (Name.consume_front("ex2.approx."))
2232 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2233 Expand =
2234 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2235 else if (Name.consume_front("fma."))
2236 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
2237 Expand = getNVVMFPArithUpgrade(Name, NVVMFMAIIDs).has_value();
2238 else if (Name.consume_front("atomic.load."))
2239 // nvvm.atomic.load.add.{f32,f64}.p
2240 // nvvm.atomic.load.{inc,dec}.32.p
2241 Expand = StringSwitch<bool>(Name)
2242 .StartsWith("add.f32.p", true)
2243 .StartsWith("add.f64.p", true)
2244 .StartsWith("inc.32.p", true)
2245 .StartsWith("dec.32.p", true)
2246 .Default(false);
2247 else if (Name.consume_front("atomic."))
2248 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2249 // nvvm.atomic.cas.gen.i.{cta,sys}
2250 Expand = StringSwitch<bool>(Name)
2251 .StartsWith("add.gen.", true)
2252 .StartsWith("exch.gen.", true)
2253 .StartsWith("max.gen.", true)
2254 .StartsWith("min.gen.", true)
2255 .StartsWith("inc.gen.", true)
2256 .StartsWith("dec.gen.", true)
2257 .StartsWith("and.gen.", true)
2258 .StartsWith("or.gen.", true)
2259 .StartsWith("xor.gen.", true)
2260 .StartsWith("cas.gen.", true)
2261 .Default(false);
2262 else if (Name.consume_front("bitcast."))
2263 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2264 Expand =
2265 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2266 else if (Name.consume_front("rotate."))
2267 // nvvm.rotate.{b32,b64,right.b64}
2268 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2269 else if (Name.consume_front("ptr.gen.to."))
2270 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2271 Expand = consumeNVVMPtrAddrSpace(Name);
2272 else if (Name.consume_front("ptr."))
2273 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2274 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2275 else if (Name.consume_front("ldg.global."))
2276 // nvvm.ldg.global.{i,p,f}
2277 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2278 Name.starts_with("p."));
2279 else
2280 Expand = StringSwitch<bool>(Name)
2281 .Case("barrier0", true)
2282 .Case("barrier.n", true)
2283 .Case("barrier.sync.cnt", true)
2284 .Case("barrier.sync", true)
2285 .Case("barrier", true)
2286 .Case("bar.sync", true)
2287 .Case("barrier0.popc", true)
2288 .Case("barrier0.and", true)
2289 .Case("barrier0.or", true)
2290 .Case("clz.ll", true)
2291 .Case("popc.ll", true)
2292 .Case("h2f", true)
2293 .Case("swap.lo.hi.b64", true)
2294 .Case("tanh.approx.f32", true)
2295 .Default(false);
2296
2297 if (Expand) {
2298 NewFn = nullptr;
2299 return true;
2300 }
2301 break; // No other 'nvvm.*'.
2302 }
2303 break;
2304 }
2305 case 'o':
2306 if (Name.starts_with("objectsize.")) {
2307 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2308 if (F->arg_size() == 2 || F->arg_size() == 3) {
2309 rename(F);
2310 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2311 Intrinsic::objectsize, Tys);
2312 return true;
2313 }
2314 }
2315 break;
2316
2317 case 'p':
2318 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2319 rename(F);
2320 NewFn = Intrinsic::getOrInsertDeclaration(
2321 F->getParent(), Intrinsic::ptr_annotation,
2322 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2323 return true;
2324 }
2325 break;
2326
2327 case 'r': {
2328 if (Name.consume_front("riscv.")) {
2329 Intrinsic::ID ID;
2330 ID = StringSwitch<Intrinsic::ID>(Name)
2331 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2332 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2333 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2334 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2335 .Default(Intrinsic::not_intrinsic);
2336 if (ID != Intrinsic::not_intrinsic) {
2337 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2338 rename(F);
2339 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2340 return true;
2341 }
2342 break; // No other applicable upgrades.
2343 }
2344
2345 ID = StringSwitch<Intrinsic::ID>(Name)
2346 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2347 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2348 .Default(Intrinsic::not_intrinsic);
2349 if (ID != Intrinsic::not_intrinsic) {
2350 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2351 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2352 rename(F);
2353 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2354 return true;
2355 }
2356 break; // No other applicable upgrades.
2357 }
2358
2359 ID = StringSwitch<Intrinsic::ID>(Name)
2360 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2361 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2362 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2363 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2364 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2365 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2366 .Default(Intrinsic::not_intrinsic);
2367 if (ID != Intrinsic::not_intrinsic) {
2368 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2369 rename(F);
2370 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2371 return true;
2372 }
2373 break; // No other applicable upgrades.
2374 }
2375
2376 // Replace llvm.riscv.clmul with llvm.clmul.
2377 if (Name == "clmul.i32" || Name == "clmul.i64") {
2378 NewFn = Intrinsic::getOrInsertDeclaration(
2379 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2380 return true;
2381 }
2382
2383 break; // No other 'riscv.*' intrinsics
2384 }
2385 } break;
2386
2387 case 's':
2388 if (Name == "stackprotectorcheck") {
2389 NewFn = nullptr;
2390 return true;
2391 }
2392 if (Name.starts_with("strip.invariant.group")) {
2393 // For clang's usage it would be safe to just drop the
2394 // strip.invariant.group, but to be conservative replace with the
2395 // stronger launder.invariant.group instead.
2396 NewFn = Intrinsic::getOrInsertDeclaration(
2397 F->getParent(), Intrinsic::launder_invariant_group,
2398 F->getReturnType());
2399 return true;
2400 }
2401 break;
2402
2403 case 't':
2404 if (Name == "thread.pointer") {
2405 NewFn = Intrinsic::getOrInsertDeclaration(
2406 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2407 return true;
2408 }
2409 break;
2410
2411 case 'v': {
2412 if (Name == "var.annotation" && F->arg_size() == 4) {
2413 rename(F);
2414 NewFn = Intrinsic::getOrInsertDeclaration(
2415 F->getParent(), Intrinsic::var_annotation,
2416 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2417 return true;
2418 }
2419 if (Name.consume_front("vector.splice")) {
2420 if (Name.starts_with(".left") || Name.starts_with(".right"))
2421 break;
2422 return true;
2423 }
2424 if (shouldUpgradeVPIntrinsic(Name))
2425 return true;
2426 break;
2427 }
2428
2429 case 'w':
2430 if (Name.consume_front("wasm.")) {
2431 Intrinsic::ID ID =
2432 StringSwitch<Intrinsic::ID>(Name)
2433 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2434 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2435 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2436 .Default(Intrinsic::not_intrinsic);
2437 if (ID != Intrinsic::not_intrinsic) {
2438 rename(F);
2439 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2440 F->getReturnType());
2441 return true;
2442 }
2443
2444 if (Name.consume_front("dot.i8x16.i7x16.")) {
2445 ID = StringSwitch<Intrinsic::ID>(Name)
2446 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2447 .Case("add.signed",
2448 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2449 .Default(Intrinsic::not_intrinsic);
2450 if (ID != Intrinsic::not_intrinsic) {
2451 rename(F);
2452 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2453 return true;
2454 }
2455 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2456 }
2457 break; // No other 'wasm.*'.
2458 }
2459 break;
2460
2461 case 'x':
2462 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2463 return true;
2464 }
2465
2466 auto *ST = dyn_cast<StructType>(F->getReturnType());
2467 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2468 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2469 // Replace return type with literal non-packed struct. Only do this for
2470 // intrinsics declared to return a struct, not for intrinsics with
2471 // overloaded return type, in which case the exact struct type will be
2472 // mangled into the name.
2473 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2474 FunctionType *FT = F->getFunctionType();
2475 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2476 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2477 std::string Name = F->getName().str();
2478 rename(F);
2479 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2480 Name, F->getParent());
2481
2482 // The new function may also need remangling.
2483 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2484 NewFn = *Result;
2485 return true;
2486 }
2487 }
2488
2489 // Remangle our intrinsic since we upgrade the mangling
2490 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2491 if (Result != std::nullopt) {
2492 NewFn = *Result;
2493 return true;
2494 }
2495
2496 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2497 return true;
2498
2499 // This may not belong here. This function is effectively being overloaded
2500 // to both detect an intrinsic which needs upgrading, and to provide the
2501 // upgraded form of the intrinsic. We should perhaps have two separate
2502 // functions for this.
2503
2504 return false;
2505}
2506
2507bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2508 bool CanUpgradeDebugIntrinsicsToRecords) {
2509 NewFn = nullptr;
2510 bool Upgraded =
2511 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2512
2513 // Upgrade intrinsic attributes. This does not change the function.
2514 if (NewFn)
2515 F = NewFn;
2516 if (Intrinsic::ID id = F->getIntrinsicID()) {
2517 // Only do this if the intrinsic signature is valid.
2518 SmallVector<Type *> OverloadTys;
2519 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2520 F->setAttributes(
2521 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2522 }
2523 return Upgraded;
2524}
2525
2526GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2527 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2528 GV->getName() == "llvm.global_dtors")) ||
2529 !GV->hasInitializer())
2530 return nullptr;
2531 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2532 if (!ATy)
2533 return nullptr;
2534 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2535 if (!STy || STy->getNumElements() != 2)
2536 return nullptr;
2537
2538 IRBuilder<> IRB(*GV->getParent());
2539 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2540 IRB.getPtrTy());
2541 Constant *Init = GV->getInitializer();
2542 unsigned N = Init->getNumOperands();
2543 std::vector<Constant *> NewCtors(N);
2544 for (unsigned i = 0; i != N; ++i) {
2545 auto Ctor = cast<Constant>(Init->getOperand(i));
2546 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2547 Ctor->getAggregateElement(1),
2548 ConstantPointerNull::get(IRB.getPtrTy()));
2549 }
2550 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2551
2552 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2553 NewInit, GV->getName());
2554}
2555
2556// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2557// to byte shuffles.
2558static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2559 unsigned Shift) {
2560 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2561 unsigned NumElts = ResultTy->getNumElements() * 8;
2562
2563 // Bitcast from a 64-bit element type to a byte element type.
2564 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2565 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2566
2567 // We'll be shuffling in zeroes.
2568 Value *Res = Constant::getNullValue(VecTy);
2569
2570 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2571 // we'll just return the zero vector.
2572 if (Shift < 16) {
2573 int Idxs[64];
2574 // 256/512-bit version is split into 2/4 16-byte lanes.
2575 for (unsigned l = 0; l != NumElts; l += 16)
2576 for (unsigned i = 0; i != 16; ++i) {
2577 unsigned Idx = NumElts + i - Shift;
2578 if (Idx < NumElts)
2579 Idx -= NumElts - 16; // end of lane, switch operand.
2580 Idxs[l + i] = Idx + l;
2581 }
2582
2583 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2584 }
2585
2586 // Bitcast back to a 64-bit element type.
2587 return Builder.CreateBitCast(Res, ResultTy, "cast");
2588}
2589
2590// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2591// to byte shuffles.
2592static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2593 unsigned Shift) {
2594 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2595 unsigned NumElts = ResultTy->getNumElements() * 8;
2596
2597 // Bitcast from a 64-bit element type to a byte element type.
2598 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2599 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2600
2601 // We'll be shuffling in zeroes.
2602 Value *Res = Constant::getNullValue(VecTy);
2603
2604 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2605 // we'll just return the zero vector.
2606 if (Shift < 16) {
2607 int Idxs[64];
2608 // 256/512-bit version is split into 2/4 16-byte lanes.
2609 for (unsigned l = 0; l != NumElts; l += 16)
2610 for (unsigned i = 0; i != 16; ++i) {
2611 unsigned Idx = i + Shift;
2612 if (Idx >= 16)
2613 Idx += NumElts - 16; // end of lane, switch operand.
2614 Idxs[l + i] = Idx + l;
2615 }
2616
2617 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2618 }
2619
2620 // Bitcast back to a 64-bit element type.
2621 return Builder.CreateBitCast(Res, ResultTy, "cast");
2622}
2623
2624static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2625 unsigned NumElts) {
2626 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2627 llvm::VectorType *MaskTy = FixedVectorType::get(
2628 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2629 Mask = Builder.CreateBitCast(Mask, MaskTy);
2630
2631 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2632 // i8 and we need to extract down to the right number of elements.
2633 if (NumElts <= 4) {
2634 int Indices[4];
2635 for (unsigned i = 0; i != NumElts; ++i)
2636 Indices[i] = i;
2637 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2638 "extract");
2639 }
2640
2641 return Mask;
2642}
2643
2644static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2645 Value *Op1) {
2646 // If the mask is all ones just emit the first operation.
2647 if (const auto *C = dyn_cast<Constant>(Mask))
2648 if (C->isAllOnesValue())
2649 return Op0;
2650
2651 Mask = getX86MaskVec(Builder, Mask,
2652 cast<FixedVectorType>(Op0->getType())->getNumElements());
2653 return Builder.CreateSelect(Mask, Op0, Op1);
2654}
2655
2656static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2657 Value *Op1) {
2658 // If the mask is all ones just emit the first operation.
2659 if (const auto *C = dyn_cast<Constant>(Mask))
2660 if (C->isAllOnesValue())
2661 return Op0;
2662
2663 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2664 Mask->getType()->getIntegerBitWidth());
2665 Mask = Builder.CreateBitCast(Mask, MaskTy);
2666 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2667 return Builder.CreateSelect(Mask, Op0, Op1);
2668}
2669
2670// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2671// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2672// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2673static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2674 Value *Op1, Value *Shift,
2675 Value *Passthru, Value *Mask,
2676 bool IsVALIGN) {
2677 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2678
2679 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2680 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2681 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2682 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2683
2684 // Mask the immediate for VALIGN.
2685 if (IsVALIGN)
2686 ShiftVal &= (NumElts - 1);
2687
2688 // If palignr is shifting the pair of vectors more than the size of two
2689 // lanes, emit zero.
2690 if (ShiftVal >= 32)
2691 return llvm::Constant::getNullValue(Op0->getType());
2692
2693 // If palignr is shifting the pair of input vectors more than one lane,
2694 // but less than two lanes, convert to shifting in zeroes.
2695 if (ShiftVal > 16) {
2696 ShiftVal -= 16;
2697 Op1 = Op0;
2698 Op0 = llvm::Constant::getNullValue(Op0->getType());
2699 }
2700
2701 int Indices[64];
2702 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2703 for (unsigned l = 0; l < NumElts; l += 16) {
2704 for (unsigned i = 0; i != 16; ++i) {
2705 unsigned Idx = ShiftVal + i;
2706 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2707 Idx += NumElts - 16; // End of lane, switch operand.
2708 Indices[l + i] = Idx + l;
2709 }
2710 }
2711
2712 Value *Align = Builder.CreateShuffleVector(
2713 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2714
2715 return emitX86Select(Builder, Mask, Align, Passthru);
2716}
2717
2718static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2719 bool ZeroMask, bool IndexForm) {
2720 Type *Ty = CI.getType();
2721 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2722 unsigned EltWidth = Ty->getScalarSizeInBits();
2723 bool IsFloat = Ty->isFPOrFPVectorTy();
2724 Intrinsic::ID IID;
2725 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2726 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2727 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2728 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2729 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2730 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2731 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2732 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2733 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2734 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2735 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2736 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2737 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2738 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2739 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2740 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2741 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2742 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2743 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2744 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2745 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2746 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2747 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2748 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2749 else if (VecWidth == 128 && EltWidth == 16)
2750 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2751 else if (VecWidth == 256 && EltWidth == 16)
2752 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2753 else if (VecWidth == 512 && EltWidth == 16)
2754 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2755 else if (VecWidth == 128 && EltWidth == 8)
2756 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2757 else if (VecWidth == 256 && EltWidth == 8)
2758 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2759 else if (VecWidth == 512 && EltWidth == 8)
2760 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2761 else
2762 llvm_unreachable("Unexpected intrinsic");
2763
2764 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2765 CI.getArgOperand(2) };
2766
2767 // If this isn't index form we need to swap operand 0 and 1.
2768 if (!IndexForm)
2769 std::swap(Args[0], Args[1]);
2770
2771 Value *V = Builder.CreateIntrinsic(IID, Args);
2772 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2773 : Builder.CreateBitCast(CI.getArgOperand(1),
2774 Ty);
2775 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2776}
2777
2778static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2779 Intrinsic::ID IID) {
2780 Type *Ty = CI.getType();
2781 Value *Op0 = CI.getOperand(0);
2782 Value *Op1 = CI.getOperand(1);
2783 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2784
2785 if (CI.arg_size() == 4) { // For masked intrinsics.
2786 Value *VecSrc = CI.getOperand(2);
2787 Value *Mask = CI.getOperand(3);
2788 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2789 }
2790 return Res;
2791}
2792
2793static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2794 bool IsRotateRight) {
2795 Type *Ty = CI.getType();
2796 Value *Src = CI.getArgOperand(0);
2797 Value *Amt = CI.getArgOperand(1);
2798
2799 // Amount may be scalar immediate, in which case create a splat vector.
2800 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2801 // we only care about the lowest log2 bits anyway.
2802 if (Amt->getType() != Ty) {
2803 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2804 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2805 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2806 }
2807
2808 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2809 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2810
2811 if (CI.arg_size() == 4) { // For masked intrinsics.
2812 Value *VecSrc = CI.getOperand(2);
2813 Value *Mask = CI.getOperand(3);
2814 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2815 }
2816 return Res;
2817}
2818
2819static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2820 bool IsSigned) {
2821 Type *Ty = CI.getType();
2822 Value *LHS = CI.getArgOperand(0);
2823 Value *RHS = CI.getArgOperand(1);
2824
2825 CmpInst::Predicate Pred;
2826 switch (Imm) {
2827 case 0x0:
2828 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2829 break;
2830 case 0x1:
2831 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2832 break;
2833 case 0x2:
2834 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2835 break;
2836 case 0x3:
2837 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2838 break;
2839 case 0x4:
2840 Pred = ICmpInst::ICMP_EQ;
2841 break;
2842 case 0x5:
2843 Pred = ICmpInst::ICMP_NE;
2844 break;
2845 case 0x6:
2846 return Constant::getNullValue(Ty); // FALSE
2847 case 0x7:
2848 return Constant::getAllOnesValue(Ty); // TRUE
2849 default:
2850 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2851 }
2852
2853 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2854 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2855 return Ext;
2856}
2857
2858static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2859 bool IsShiftRight, bool ZeroMask) {
2860 Type *Ty = CI.getType();
2861 Value *Op0 = CI.getArgOperand(0);
2862 Value *Op1 = CI.getArgOperand(1);
2863 Value *Amt = CI.getArgOperand(2);
2864
2865 if (IsShiftRight)
2866 std::swap(Op0, Op1);
2867
2868 // Amount may be scalar immediate, in which case create a splat vector.
2869 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2870 // we only care about the lowest log2 bits anyway.
2871 if (Amt->getType() != Ty) {
2872 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2873 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2874 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2875 }
2876
2877 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2878 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2879
2880 unsigned NumArgs = CI.arg_size();
2881 if (NumArgs >= 4) { // For masked intrinsics.
2882 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2883 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2884 CI.getArgOperand(0);
2885 Value *Mask = CI.getOperand(NumArgs - 1);
2886 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2887 }
2888 return Res;
2889}
2890
2891static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2892 Value *Mask, bool Aligned) {
2893 const Align Alignment =
2894 Aligned
2895 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2896 : Align(1);
2897
2898 // If the mask is all ones just emit a regular store.
2899 if (const auto *C = dyn_cast<Constant>(Mask))
2900 if (C->isAllOnesValue())
2901 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2902
2903 // Convert the mask from an integer type to a vector of i1.
2904 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2905 Mask = getX86MaskVec(Builder, Mask, NumElts);
2906 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2907}
2908
2909static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2910 Value *Passthru, Value *Mask, bool Aligned) {
2911 Type *ValTy = Passthru->getType();
2912 const Align Alignment =
2913 Aligned
2914 ? Align(
2915 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2916 8)
2917 : Align(1);
2918
2919 // If the mask is all ones just emit a regular store.
2920 if (const auto *C = dyn_cast<Constant>(Mask))
2921 if (C->isAllOnesValue())
2922 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2923
2924 // Convert the mask from an integer type to a vector of i1.
2925 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2926 Mask = getX86MaskVec(Builder, Mask, NumElts);
2927 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2928}
2929
2930static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2931 Type *Ty = CI.getType();
2932 Value *Op0 = CI.getArgOperand(0);
2933 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2934 {Op0, Builder.getInt1(false)});
2935 if (CI.arg_size() == 3)
2936 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2937 return Res;
2938}
2939
2940static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2941 Type *Ty = CI.getType();
2942
2943 // Arguments have a vXi32 type so cast to vXi64.
2944 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2945 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2946
2947 if (IsSigned) {
2948 // Shift left then arithmetic shift right.
2949 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2950 LHS = Builder.CreateShl(LHS, ShiftAmt);
2951 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2952 RHS = Builder.CreateShl(RHS, ShiftAmt);
2953 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2954 } else {
2955 // Clear the upper bits.
2956 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2957 LHS = Builder.CreateAnd(LHS, Mask);
2958 RHS = Builder.CreateAnd(RHS, Mask);
2959 }
2960
2961 Value *Res = Builder.CreateMul(LHS, RHS);
2962
2963 if (CI.arg_size() == 4)
2964 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2965
2966 return Res;
2967}
2968
2969// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2970static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2971 Value *Mask) {
2972 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2973 if (Mask) {
2974 const auto *C = dyn_cast<Constant>(Mask);
2975 if (!C || !C->isAllOnesValue())
2976 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2977 }
2978
2979 if (NumElts < 8) {
2980 int Indices[8];
2981 for (unsigned i = 0; i != NumElts; ++i)
2982 Indices[i] = i;
2983 for (unsigned i = NumElts; i != 8; ++i)
2984 Indices[i] = NumElts + i % NumElts;
2985 Vec = Builder.CreateShuffleVector(Vec,
2986 Constant::getNullValue(Vec->getType()),
2987 Indices);
2988 }
2989 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2990}
2991
2992static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2993 unsigned CC, bool Signed) {
2994 Value *Op0 = CI.getArgOperand(0);
2995 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2996
2997 Value *Cmp;
2998 if (CC == 3) {
2999 Cmp = Constant::getNullValue(
3000 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3001 } else if (CC == 7) {
3002 Cmp = Constant::getAllOnesValue(
3003 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3004 } else {
3005 ICmpInst::Predicate Pred;
3006 switch (CC) {
3007 default: llvm_unreachable("Unknown condition code");
3008 case 0: Pred = ICmpInst::ICMP_EQ; break;
3009 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
3010 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
3011 case 4: Pred = ICmpInst::ICMP_NE; break;
3012 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
3013 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
3014 }
3015 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
3016 }
3017
3018 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
3019
3020 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
3021}
3022
3023// Replace a masked intrinsic with an older unmasked intrinsic.
3024static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
3025 Intrinsic::ID IID) {
3026 Value *Rep =
3027 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
3028 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
3029}
3030
3031static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3032 Value* A = CI.getArgOperand(0);
3033 Value* B = CI.getArgOperand(1);
3034 Value* Src = CI.getArgOperand(2);
3035 Value* Mask = CI.getArgOperand(3);
3036
3037 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
3038 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3039 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
3040 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
3041 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3042 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3043}
3044
3045static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3046 Value* Op = CI.getArgOperand(0);
3047 Type* ReturnOp = CI.getType();
3048 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3049 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3050 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3051}
3052
3053// Replace intrinsic with unmasked version and a select.
3054static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3055 CallBase &CI, Value *&Rep) {
3056 Name = Name.substr(12); // Remove avx512.mask.
3057
3058 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3059 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3060 Intrinsic::ID IID;
3061 if (Name.starts_with("max.p")) {
3062 if (VecWidth == 128 && EltWidth == 32)
3063 IID = Intrinsic::x86_sse_max_ps;
3064 else if (VecWidth == 128 && EltWidth == 64)
3065 IID = Intrinsic::x86_sse2_max_pd;
3066 else if (VecWidth == 256 && EltWidth == 32)
3067 IID = Intrinsic::x86_avx_max_ps_256;
3068 else if (VecWidth == 256 && EltWidth == 64)
3069 IID = Intrinsic::x86_avx_max_pd_256;
3070 else
3071 llvm_unreachable("Unexpected intrinsic");
3072 } else if (Name.starts_with("min.p")) {
3073 if (VecWidth == 128 && EltWidth == 32)
3074 IID = Intrinsic::x86_sse_min_ps;
3075 else if (VecWidth == 128 && EltWidth == 64)
3076 IID = Intrinsic::x86_sse2_min_pd;
3077 else if (VecWidth == 256 && EltWidth == 32)
3078 IID = Intrinsic::x86_avx_min_ps_256;
3079 else if (VecWidth == 256 && EltWidth == 64)
3080 IID = Intrinsic::x86_avx_min_pd_256;
3081 else
3082 llvm_unreachable("Unexpected intrinsic");
3083 } else if (Name.starts_with("pshuf.b.")) {
3084 if (VecWidth == 128)
3085 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3086 else if (VecWidth == 256)
3087 IID = Intrinsic::x86_avx2_pshuf_b;
3088 else if (VecWidth == 512)
3089 IID = Intrinsic::x86_avx512_pshuf_b_512;
3090 else
3091 llvm_unreachable("Unexpected intrinsic");
3092 } else if (Name.starts_with("pmul.hr.sw.")) {
3093 if (VecWidth == 128)
3094 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3095 else if (VecWidth == 256)
3096 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3097 else if (VecWidth == 512)
3098 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3099 else
3100 llvm_unreachable("Unexpected intrinsic");
3101 } else if (Name.starts_with("pmulh.w")) {
3102 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3103 "Unexpected intrinsic");
3104 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::smulh);
3105 return true;
3106 } else if (Name.starts_with("pmulhu.w")) {
3107 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3108 "Unexpected intrinsic");
3109 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::umulh);
3110 return true;
3111 } else if (Name.starts_with("pmaddw.d.")) {
3112 if (VecWidth == 128)
3113 IID = Intrinsic::x86_sse2_pmadd_wd;
3114 else if (VecWidth == 256)
3115 IID = Intrinsic::x86_avx2_pmadd_wd;
3116 else if (VecWidth == 512)
3117 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3118 else
3119 llvm_unreachable("Unexpected intrinsic");
3120 } else if (Name.starts_with("pmaddubs.w.")) {
3121 if (VecWidth == 128)
3122 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3123 else if (VecWidth == 256)
3124 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3125 else if (VecWidth == 512)
3126 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3127 else
3128 llvm_unreachable("Unexpected intrinsic");
3129 } else if (Name.starts_with("packsswb.")) {
3130 if (VecWidth == 128)
3131 IID = Intrinsic::x86_sse2_packsswb_128;
3132 else if (VecWidth == 256)
3133 IID = Intrinsic::x86_avx2_packsswb;
3134 else if (VecWidth == 512)
3135 IID = Intrinsic::x86_avx512_packsswb_512;
3136 else
3137 llvm_unreachable("Unexpected intrinsic");
3138 } else if (Name.starts_with("packssdw.")) {
3139 if (VecWidth == 128)
3140 IID = Intrinsic::x86_sse2_packssdw_128;
3141 else if (VecWidth == 256)
3142 IID = Intrinsic::x86_avx2_packssdw;
3143 else if (VecWidth == 512)
3144 IID = Intrinsic::x86_avx512_packssdw_512;
3145 else
3146 llvm_unreachable("Unexpected intrinsic");
3147 } else if (Name.starts_with("packuswb.")) {
3148 if (VecWidth == 128)
3149 IID = Intrinsic::x86_sse2_packuswb_128;
3150 else if (VecWidth == 256)
3151 IID = Intrinsic::x86_avx2_packuswb;
3152 else if (VecWidth == 512)
3153 IID = Intrinsic::x86_avx512_packuswb_512;
3154 else
3155 llvm_unreachable("Unexpected intrinsic");
3156 } else if (Name.starts_with("packusdw.")) {
3157 if (VecWidth == 128)
3158 IID = Intrinsic::x86_sse41_packusdw;
3159 else if (VecWidth == 256)
3160 IID = Intrinsic::x86_avx2_packusdw;
3161 else if (VecWidth == 512)
3162 IID = Intrinsic::x86_avx512_packusdw_512;
3163 else
3164 llvm_unreachable("Unexpected intrinsic");
3165 } else if (Name.starts_with("vpermilvar.")) {
3166 if (VecWidth == 128 && EltWidth == 32)
3167 IID = Intrinsic::x86_avx_vpermilvar_ps;
3168 else if (VecWidth == 128 && EltWidth == 64)
3169 IID = Intrinsic::x86_avx_vpermilvar_pd;
3170 else if (VecWidth == 256 && EltWidth == 32)
3171 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3172 else if (VecWidth == 256 && EltWidth == 64)
3173 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3174 else if (VecWidth == 512 && EltWidth == 32)
3175 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3176 else if (VecWidth == 512 && EltWidth == 64)
3177 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3178 else
3179 llvm_unreachable("Unexpected intrinsic");
3180 } else if (Name == "cvtpd2dq.256") {
3181 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3182 } else if (Name == "cvtpd2ps.256") {
3183 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3184 } else if (Name == "cvttpd2dq.256") {
3185 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3186 } else if (Name == "cvttps2dq.128") {
3187 IID = Intrinsic::x86_sse2_cvttps2dq;
3188 } else if (Name == "cvttps2dq.256") {
3189 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3190 } else if (Name.starts_with("permvar.")) {
3191 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3192 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3193 IID = Intrinsic::x86_avx2_permps;
3194 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3195 IID = Intrinsic::x86_avx2_permd;
3196 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3197 IID = Intrinsic::x86_avx512_permvar_df_256;
3198 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3199 IID = Intrinsic::x86_avx512_permvar_di_256;
3200 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3201 IID = Intrinsic::x86_avx512_permvar_sf_512;
3202 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3203 IID = Intrinsic::x86_avx512_permvar_si_512;
3204 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3205 IID = Intrinsic::x86_avx512_permvar_df_512;
3206 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3207 IID = Intrinsic::x86_avx512_permvar_di_512;
3208 else if (VecWidth == 128 && EltWidth == 16)
3209 IID = Intrinsic::x86_avx512_permvar_hi_128;
3210 else if (VecWidth == 256 && EltWidth == 16)
3211 IID = Intrinsic::x86_avx512_permvar_hi_256;
3212 else if (VecWidth == 512 && EltWidth == 16)
3213 IID = Intrinsic::x86_avx512_permvar_hi_512;
3214 else if (VecWidth == 128 && EltWidth == 8)
3215 IID = Intrinsic::x86_avx512_permvar_qi_128;
3216 else if (VecWidth == 256 && EltWidth == 8)
3217 IID = Intrinsic::x86_avx512_permvar_qi_256;
3218 else if (VecWidth == 512 && EltWidth == 8)
3219 IID = Intrinsic::x86_avx512_permvar_qi_512;
3220 else
3221 llvm_unreachable("Unexpected intrinsic");
3222 } else if (Name.starts_with("dbpsadbw.")) {
3223 if (VecWidth == 128)
3224 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3225 else if (VecWidth == 256)
3226 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3227 else if (VecWidth == 512)
3228 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3229 else
3230 llvm_unreachable("Unexpected intrinsic");
3231 } else if (Name.starts_with("pmultishift.qb.")) {
3232 if (VecWidth == 128)
3233 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3234 else if (VecWidth == 256)
3235 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3236 else if (VecWidth == 512)
3237 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3238 else
3239 llvm_unreachable("Unexpected intrinsic");
3240 } else if (Name.starts_with("conflict.")) {
3241 if (Name[9] == 'd' && VecWidth == 128)
3242 IID = Intrinsic::x86_avx512_conflict_d_128;
3243 else if (Name[9] == 'd' && VecWidth == 256)
3244 IID = Intrinsic::x86_avx512_conflict_d_256;
3245 else if (Name[9] == 'd' && VecWidth == 512)
3246 IID = Intrinsic::x86_avx512_conflict_d_512;
3247 else if (Name[9] == 'q' && VecWidth == 128)
3248 IID = Intrinsic::x86_avx512_conflict_q_128;
3249 else if (Name[9] == 'q' && VecWidth == 256)
3250 IID = Intrinsic::x86_avx512_conflict_q_256;
3251 else if (Name[9] == 'q' && VecWidth == 512)
3252 IID = Intrinsic::x86_avx512_conflict_q_512;
3253 else
3254 llvm_unreachable("Unexpected intrinsic");
3255 } else if (Name.starts_with("pavg.")) {
3256 if (Name[5] == 'b' && VecWidth == 128)
3257 IID = Intrinsic::x86_sse2_pavg_b;
3258 else if (Name[5] == 'b' && VecWidth == 256)
3259 IID = Intrinsic::x86_avx2_pavg_b;
3260 else if (Name[5] == 'b' && VecWidth == 512)
3261 IID = Intrinsic::x86_avx512_pavg_b_512;
3262 else if (Name[5] == 'w' && VecWidth == 128)
3263 IID = Intrinsic::x86_sse2_pavg_w;
3264 else if (Name[5] == 'w' && VecWidth == 256)
3265 IID = Intrinsic::x86_avx2_pavg_w;
3266 else if (Name[5] == 'w' && VecWidth == 512)
3267 IID = Intrinsic::x86_avx512_pavg_w_512;
3268 else
3269 llvm_unreachable("Unexpected intrinsic");
3270 } else
3271 return false;
3272
3273 SmallVector<Value *, 4> Args(CI.args());
3274 Args.pop_back();
3275 Args.pop_back();
3276 Rep = Builder.CreateIntrinsic(IID, Args);
3277 unsigned NumArgs = CI.arg_size();
3278 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3279 CI.getArgOperand(NumArgs - 2));
3280 return true;
3281}
3282
3283/// Upgrade comment in call to inline asm that represents an objc retain release
3284/// marker.
3285void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3286 size_t Pos;
3287 if (AsmStr->find("mov\tfp") == 0 &&
3288 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3289 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3290 AsmStr->replace(Pos, 1, ";");
3291 }
3292}
3293
3294static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3295 StringRef Name,
3296 const Intrinsic::ID (&IIDs)[2][3]) {
3297 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3298 assert(Result && "unsupported nvvm.{add,mul,fma}.* intrinsic");
3299 auto [IID, RoundingMode] = *Result;
3300
3301 // The legacy bf16 variants used integer storage types for the bf16
3302 // operands; bitcast them to the floating-point types of the new
3303 // overloaded intrinsic.
3304 Type *Ty = CI->getType();
3305 const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
3306 if (IsLegacyBF16Storage)
3307 Ty = Name.ends_with("bf16x2")
3308 ? FixedVectorType::get(Builder.getBFloatTy(), 2)
3309 : static_cast<Type *>(Builder.getBFloatTy());
3310
3311 SmallVector<Value *, 4> Args;
3312 for (Value *Arg : CI->args())
3313 Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty) : Arg);
3314 Args.push_back(Builder.getInt32(static_cast<int>(RoundingMode)));
3315
3316 Value *Rep = Builder.CreateIntrinsic(Ty, IID, Args);
3317 return IsLegacyBF16Storage ? Builder.CreateBitCast(Rep, CI->getType()) : Rep;
3318}
3319
3320static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3321 Function *F, IRBuilder<> &Builder) {
3322 Value *Rep = nullptr;
3323
3324 if (Name == "abs.i" || Name == "abs.ll") {
3325 Value *Arg = CI->getArgOperand(0);
3326 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3327 {Arg, Builder.getTrue()},
3328 /*FMFSource=*/nullptr, "abs");
3329 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3330 Type *Ty = (Name == "abs.bf16")
3331 ? Builder.getBFloatTy()
3332 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3333 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3334 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3335 Rep = Builder.CreateBitCast(Abs, CI->getType());
3336 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3337 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3338 : Intrinsic::nvvm_fabs;
3339 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3340 } else if (Name.consume_front("add.")) {
3341 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3342 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFAddIIDs);
3343 } else if (Name.consume_front("mul.")) {
3344 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3345 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMulIIDs);
3346 } else if (Name.consume_front("ex2.approx.")) {
3347 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3348 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3349 : Intrinsic::nvvm_ex2_approx;
3350 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3351 } else if (Name.consume_front("fma.")) {
3352 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
3353 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMAIIDs);
3354 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3355 Name.starts_with("atomic.load.add.f64.p")) {
3356 Value *Ptr = CI->getArgOperand(0);
3357 Value *Val = CI->getArgOperand(1);
3358 Rep = Builder.CreateAtomicRMW(
3359 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3360 CI->getContext().getOrInsertSyncScopeID("device"));
3361 cast<Instruction>(Rep)->setMetadata(
3362 LLVMContext::MD_atomic_ignore_denormal_mode,
3363 MDNode::get(CI->getContext(), {}));
3364 // The default scope for atomic.load.* intrinsics is device
3365 // (= gpu scope in ptx), but the default LLVM atomic scope is
3366 // "system"
3367 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3368 Name.starts_with("atomic.load.dec.32.p")) {
3369 Value *Ptr = CI->getArgOperand(0);
3370 Value *Val = CI->getArgOperand(1);
3371 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3372 : AtomicRMWInst::UDecWrap;
3373 Rep = Builder.CreateAtomicRMW(
3374 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3375 CI->getContext().getOrInsertSyncScopeID("device"));
3376 // See comment above.
3377 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3378 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3379 StringRef Op = Name.substr(StringRef("atomic.").size());
3380 Value *Ptr = CI->getArgOperand(0);
3381 Value *Val = CI->getArgOperand(1);
3382 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3383 Op.contains(".cta.") ? "block" : "");
3384 if (Op.starts_with("cas.")) {
3385 Value *New = CI->getArgOperand(2);
3386 Value *Pair = Builder.CreateAtomicCmpXchg(
3387 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3388 AtomicOrdering::Monotonic, SSID);
3389 Rep = Builder.CreateExtractValue(Pair, 0);
3390 } else {
3391 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3392 // because we were actually missing those intrinsics!
3393 AtomicRMWInst::BinOp BinOp =
3394 StringSwitch<AtomicRMWInst::BinOp>(Op)
3395 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3396 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3397 .StartsWith("exch.", AtomicRMWInst::Xchg)
3398 .StartsWith("max.", AtomicRMWInst::Max)
3399 .StartsWith("min.", AtomicRMWInst::Min)
3400 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3401 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3402 .StartsWith("and.", AtomicRMWInst::And)
3403 .StartsWith("or.", AtomicRMWInst::Or)
3404 .StartsWith("xor.", AtomicRMWInst::Xor)
3405 .Default(AtomicRMWInst::BAD_BINOP);
3406 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3407 "unexpected nvvm scoped atomic intrinsic");
3408 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3409 AtomicOrdering::Monotonic, SSID);
3410 if (BinOp == AtomicRMWInst::FAdd) {
3411 cast<Instruction>(Rep)->setMetadata(
3412 LLVMContext::MD_atomic_ignore_denormal_mode,
3413 MDNode::get(CI->getContext(), {}));
3414 }
3415 }
3416 } else if (Name == "clz.ll") {
3417 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3418 Value *Arg = CI->getArgOperand(0);
3419 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3420 {Arg, Builder.getFalse()},
3421 /*FMFSource=*/nullptr, "ctlz");
3422 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3423 } else if (Name == "popc.ll") {
3424 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3425 // i64.
3426 Value *Arg = CI->getArgOperand(0);
3427 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3428 Arg, /*FMFSource=*/nullptr, "ctpop");
3429 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3430 } else if (Name == "h2f") {
3431 Value *Cast =
3432 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3433 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3434 } else if (Name.consume_front("bitcast.") &&
3435 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3436 Name == "d2ll")) {
3437 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3438 } else if (Name == "rotate.b32") {
3439 Value *Arg = CI->getOperand(0);
3440 Value *ShiftAmt = CI->getOperand(1);
3441 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3442 {Arg, Arg, ShiftAmt});
3443 } else if (Name == "rotate.b64") {
3444 Type *Int64Ty = Builder.getInt64Ty();
3445 Value *Arg = CI->getOperand(0);
3446 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3447 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3448 {Arg, Arg, ZExtShiftAmt});
3449 } else if (Name == "rotate.right.b64") {
3450 Type *Int64Ty = Builder.getInt64Ty();
3451 Value *Arg = CI->getOperand(0);
3452 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3453 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3454 {Arg, Arg, ZExtShiftAmt});
3455 } else if (Name == "swap.lo.hi.b64") {
3456 Type *Int64Ty = Builder.getInt64Ty();
3457 Value *Arg = CI->getOperand(0);
3458 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3459 {Arg, Arg, Builder.getInt64(32)});
3460 } else if ((Name.consume_front("ptr.gen.to.") &&
3461 consumeNVVMPtrAddrSpace(Name)) ||
3462 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3463 Name.starts_with(".to.gen"))) {
3464 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3465 } else if (Name.consume_front("ldg.global")) {
3466 Value *Ptr = CI->getArgOperand(0);
3467 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3468 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3469 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3470 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3471 MDNode *MD = MDNode::get(Builder.getContext(), {});
3472 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3473 return LD;
3474 } else if (Name == "tanh.approx.f32") {
3475 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3476 FastMathFlags FMF;
3477 FMF.setApproxFunc();
3478 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3479 FMF);
3480 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3481 Value *Arg =
3482 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3483 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3484 {}, {Arg});
3485 } else if (Name == "barrier") {
3486 Rep = Builder.CreateIntrinsic(
3487 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3488 {CI->getArgOperand(0), CI->getArgOperand(1)});
3489 } else if (Name == "barrier.sync") {
3490 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3491 {CI->getArgOperand(0)});
3492 } else if (Name == "barrier.sync.cnt") {
3493 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3494 {CI->getArgOperand(0), CI->getArgOperand(1)});
3495 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3496 Name == "barrier0.or") {
3497 Value *C = CI->getArgOperand(0);
3498 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3499
3500 Intrinsic::ID IID =
3501 StringSwitch<Intrinsic::ID>(Name)
3502 .Case("barrier0.popc",
3503 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3504 .Case("barrier0.and",
3505 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3506 .Case("barrier0.or",
3507 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3508 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3509 Rep = Builder.CreateZExt(Bar, CI->getType());
3510 } else {
3511 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3512 if (IID != Intrinsic::not_intrinsic &&
3513 isLegacyNVPTXBF16IntSignature(F, IID)) {
3514 rename(F);
3515 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3516 SmallVector<Value *, 2> Args;
3517 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3518 Value *Arg = CI->getArgOperand(I);
3519 Type *OldType = Arg->getType();
3520 Type *NewType = NewFn->getArg(I)->getType();
3521 Args.push_back(
3522 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3523 ? Builder.CreateBitCast(Arg, NewType)
3524 : Arg);
3525 }
3526 Rep = Builder.CreateCall(NewFn, Args);
3527 if (F->getReturnType()->isIntegerTy())
3528 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3529 }
3530 }
3531
3532 return Rep;
3533}
3534
3535static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3536 IRBuilder<> &Builder) {
3537 LLVMContext &C = F->getContext();
3538 Value *Rep = nullptr;
3539
3540 if (Name.starts_with("sse4a.movnt.")) {
3541 SmallVector<Metadata *, 1> Elts;
3542 Elts.push_back(
3543 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3544 MDNode *Node = MDNode::get(C, Elts);
3545
3546 Value *Arg0 = CI->getArgOperand(0);
3547 Value *Arg1 = CI->getArgOperand(1);
3548
3549 // Nontemporal (unaligned) store of the 0'th element of the float/double
3550 // vector.
3551 Value *Extract =
3552 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3553
3554 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3555 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3556 } else if (Name.starts_with("avx.movnt.") ||
3557 Name.starts_with("avx512.storent.")) {
3558 SmallVector<Metadata *, 1> Elts;
3559 Elts.push_back(
3560 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3561 MDNode *Node = MDNode::get(C, Elts);
3562
3563 Value *Arg0 = CI->getArgOperand(0);
3564 Value *Arg1 = CI->getArgOperand(1);
3565
3566 StoreInst *SI = Builder.CreateAlignedStore(
3567 Arg1, Arg0,
3568 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3569 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3570 } else if (Name == "sse2.storel.dq") {
3571 Value *Arg0 = CI->getArgOperand(0);
3572 Value *Arg1 = CI->getArgOperand(1);
3573
3574 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3575 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3576 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3577 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3578 } else if (Name.starts_with("sse.storeu.") ||
3579 Name.starts_with("sse2.storeu.") ||
3580 Name.starts_with("avx.storeu.")) {
3581 Value *Arg0 = CI->getArgOperand(0);
3582 Value *Arg1 = CI->getArgOperand(1);
3583 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3584 } else if (Name == "avx512.mask.store.ss") {
3585 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3586 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3587 Mask, false);
3588 } else if (Name.starts_with("avx512.mask.store")) {
3589 // "avx512.mask.storeu." or "avx512.mask.store."
3590 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3591 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3592 CI->getArgOperand(2), Aligned);
3593 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3594 // Upgrade packed integer vector compare intrinsics to compare instructions.
3595 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3596 bool CmpEq = Name[9] == 'e';
3597 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3598 CI->getArgOperand(0), CI->getArgOperand(1));
3599 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3600 } else if (Name.starts_with("avx512.broadcastm")) {
3601 Type *ExtTy = Type::getInt32Ty(C);
3602 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3603 ExtTy = Type::getInt64Ty(C);
3604 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3605 ExtTy->getPrimitiveSizeInBits();
3606 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3607 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3608 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3609 Value *Vec = CI->getArgOperand(0);
3610 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3611 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3612 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3613 } else if (Name.starts_with("avx.sqrt.p") ||
3614 Name.starts_with("sse2.sqrt.p") ||
3615 Name.starts_with("sse.sqrt.p")) {
3616 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3617 {CI->getArgOperand(0)});
3618 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3619 if (CI->arg_size() == 4 &&
3620 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3621 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3622 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3623 : Intrinsic::x86_avx512_sqrt_pd_512;
3624
3625 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3626 Rep = Builder.CreateIntrinsic(IID, Args);
3627 } else {
3628 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3629 {CI->getArgOperand(0)});
3630 }
3631 Rep =
3632 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3633 } else if (Name.starts_with("avx512.ptestm") ||
3634 Name.starts_with("avx512.ptestnm")) {
3635 Value *Op0 = CI->getArgOperand(0);
3636 Value *Op1 = CI->getArgOperand(1);
3637 Value *Mask = CI->getArgOperand(2);
3638 Rep = Builder.CreateAnd(Op0, Op1);
3639 llvm::Type *Ty = Op0->getType();
3640 Value *Zero = llvm::Constant::getNullValue(Ty);
3641 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3642 ? ICmpInst::ICMP_NE
3643 : ICmpInst::ICMP_EQ;
3644 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3645 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3646 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3647 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3648 ->getNumElements();
3649 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3650 Rep =
3651 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3652 } else if (Name.starts_with("avx512.kunpck")) {
3653 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3654 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3655 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3656 int Indices[64];
3657 for (unsigned i = 0; i != NumElts; ++i)
3658 Indices[i] = i;
3659
3660 // First extract half of each vector. This gives better codegen than
3661 // doing it in a single shuffle.
3662 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3663 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3664 // Concat the vectors.
3665 // NOTE: Operands have to be swapped to match intrinsic definition.
3666 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3667 Rep = Builder.CreateBitCast(Rep, CI->getType());
3668 } else if (Name == "avx512.kand.w") {
3669 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3670 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3671 Rep = Builder.CreateAnd(LHS, RHS);
3672 Rep = Builder.CreateBitCast(Rep, CI->getType());
3673 } else if (Name == "avx512.kandn.w") {
3674 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3675 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3676 LHS = Builder.CreateNot(LHS);
3677 Rep = Builder.CreateAnd(LHS, RHS);
3678 Rep = Builder.CreateBitCast(Rep, CI->getType());
3679 } else if (Name == "avx512.kor.w") {
3680 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3681 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3682 Rep = Builder.CreateOr(LHS, RHS);
3683 Rep = Builder.CreateBitCast(Rep, CI->getType());
3684 } else if (Name == "avx512.kxor.w") {
3685 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3686 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3687 Rep = Builder.CreateXor(LHS, RHS);
3688 Rep = Builder.CreateBitCast(Rep, CI->getType());
3689 } else if (Name == "avx512.kxnor.w") {
3690 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3691 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3692 LHS = Builder.CreateNot(LHS);
3693 Rep = Builder.CreateXor(LHS, RHS);
3694 Rep = Builder.CreateBitCast(Rep, CI->getType());
3695 } else if (Name == "avx512.knot.w") {
3696 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3697 Rep = Builder.CreateNot(Rep);
3698 Rep = Builder.CreateBitCast(Rep, CI->getType());
3699 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3700 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3701 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3702 Rep = Builder.CreateOr(LHS, RHS);
3703 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3704 Value *C;
3705 if (Name[14] == 'c')
3706 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3707 else
3708 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3709 Rep = Builder.CreateICmpEQ(Rep, C);
3710 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3711 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3712 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3713 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3714 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3715 Type *I32Ty = Type::getInt32Ty(C);
3716 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3717 ConstantInt::get(I32Ty, 0));
3718 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3719 ConstantInt::get(I32Ty, 0));
3720 Value *EltOp;
3721 if (Name.contains(".add."))
3722 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3723 else if (Name.contains(".sub."))
3724 EltOp = Builder.CreateFSub(Elt0, Elt1);
3725 else if (Name.contains(".mul."))
3726 EltOp = Builder.CreateFMul(Elt0, Elt1);
3727 else
3728 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3729 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3730 ConstantInt::get(I32Ty, 0));
3731 } else if (Name.starts_with("avx512.mask.pcmp")) {
3732 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3733 bool CmpEq = Name[16] == 'e';
3734 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3735 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3736 Type *OpTy = CI->getArgOperand(0)->getType();
3737 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3738 Intrinsic::ID IID;
3739 switch (VecWidth) {
3740 default:
3741 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3742 break;
3743 case 128:
3744 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3745 break;
3746 case 256:
3747 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3748 break;
3749 case 512:
3750 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3751 break;
3752 }
3753
3754 Rep =
3755 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3756 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3757 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3758 Type *OpTy = CI->getArgOperand(0)->getType();
3759 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3760 unsigned EltWidth = OpTy->getScalarSizeInBits();
3761 Intrinsic::ID IID;
3762 if (VecWidth == 128 && EltWidth == 32)
3763 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3764 else if (VecWidth == 256 && EltWidth == 32)
3765 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3766 else if (VecWidth == 512 && EltWidth == 32)
3767 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3768 else if (VecWidth == 128 && EltWidth == 64)
3769 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3770 else if (VecWidth == 256 && EltWidth == 64)
3771 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3772 else if (VecWidth == 512 && EltWidth == 64)
3773 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3774 else
3775 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3776
3777 Rep =
3778 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3779 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3780 } else if (Name.starts_with("avx512.cmp.p")) {
3781 SmallVector<Value *, 4> Args(CI->args());
3782 Type *OpTy = Args[0]->getType();
3783 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3784 unsigned EltWidth = OpTy->getScalarSizeInBits();
3785 Intrinsic::ID IID;
3786 if (VecWidth == 128 && EltWidth == 32)
3787 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3788 else if (VecWidth == 256 && EltWidth == 32)
3789 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3790 else if (VecWidth == 512 && EltWidth == 32)
3791 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3792 else if (VecWidth == 128 && EltWidth == 64)
3793 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3794 else if (VecWidth == 256 && EltWidth == 64)
3795 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3796 else if (VecWidth == 512 && EltWidth == 64)
3797 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3798 else
3799 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3800
3801 Value *Mask = Constant::getAllOnesValue(CI->getType());
3802 if (VecWidth == 512)
3803 std::swap(Mask, Args.back());
3804 Args.push_back(Mask);
3805
3806 Rep = Builder.CreateIntrinsic(IID, Args);
3807 } else if (Name.starts_with("avx512.mask.cmp.")) {
3808 // Integer compare intrinsics.
3809 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3810 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3811 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3812 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3813 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3814 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3815 Name.starts_with("avx512.cvtw2mask.") ||
3816 Name.starts_with("avx512.cvtd2mask.") ||
3817 Name.starts_with("avx512.cvtq2mask.")) {
3818 Value *Op = CI->getArgOperand(0);
3819 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3820 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3821 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3822 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3823 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3824 Name.starts_with("avx512.mask.pabs")) {
3825 Rep = upgradeAbs(Builder, *CI);
3826 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3827 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3828 Name.starts_with("avx512.mask.pmaxs")) {
3829 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3830 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3831 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3832 Name.starts_with("avx512.mask.pmaxu")) {
3833 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3834 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3835 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3836 Name.starts_with("avx512.mask.pmins")) {
3837 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3838 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3839 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3840 Name.starts_with("avx512.mask.pminu")) {
3841 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3842 } else if (Name == "sse2.pmulh.w" || Name.starts_with("avx2.pmulh.w") ||
3843 Name.starts_with("avx512.pmulh.w")) {
3844 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smulh);
3845 } else if (Name == "sse2.pmulhu.w" || Name.starts_with("avx2.pmulhu.w") ||
3846 Name.starts_with("avx512.pmulhu.w")) {
3847 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umulh);
3848 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3849 Name == "avx512.pmulu.dq.512" ||
3850 Name.starts_with("avx512.mask.pmulu.dq.")) {
3851 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3852 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3853 Name == "avx512.pmul.dq.512" ||
3854 Name.starts_with("avx512.mask.pmul.dq.")) {
3855 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3856 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3857 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3858 Rep =
3859 Builder.CreateSIToFP(CI->getArgOperand(1),
3860 cast<VectorType>(CI->getType())->getElementType());
3861 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3862 } else if (Name == "avx512.cvtusi2sd") {
3863 Rep =
3864 Builder.CreateUIToFP(CI->getArgOperand(1),
3865 cast<VectorType>(CI->getType())->getElementType());
3866 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3867 } else if (Name == "sse2.cvtss2sd") {
3868 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3869 Rep = Builder.CreateFPExt(
3870 Rep, cast<VectorType>(CI->getType())->getElementType());
3871 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3872 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3873 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3874 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3875 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3876 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3877 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3878 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3879 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3880 Name == "avx512.mask.cvtqq2ps.256" ||
3881 Name == "avx512.mask.cvtqq2ps.512" ||
3882 Name == "avx512.mask.cvtuqq2ps.256" ||
3883 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3884 Name == "avx.cvt.ps2.pd.256" ||
3885 Name == "avx512.mask.cvtps2pd.128" ||
3886 Name == "avx512.mask.cvtps2pd.256") {
3887 auto *DstTy = cast<FixedVectorType>(CI->getType());
3888 Rep = CI->getArgOperand(0);
3889 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3890
3891 unsigned NumDstElts = DstTy->getNumElements();
3892 if (NumDstElts < SrcTy->getNumElements()) {
3893 assert(NumDstElts == 2 && "Unexpected vector size");
3894 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3895 }
3896
3897 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3898 bool IsUnsigned = Name.contains("cvtu");
3899 if (IsPS2PD)
3900 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3901 else if (CI->arg_size() == 4 &&
3902 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3903 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3904 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3905 : Intrinsic::x86_avx512_sitofp_round;
3906 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3907 {Rep, CI->getArgOperand(3)});
3908 } else {
3909 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3910 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3911 }
3912
3913 if (CI->arg_size() >= 3)
3914 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3915 CI->getArgOperand(1));
3916 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3917 Name.starts_with("vcvtph2ps.")) {
3918 auto *DstTy = cast<FixedVectorType>(CI->getType());
3919 Rep = CI->getArgOperand(0);
3920 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3921 unsigned NumDstElts = DstTy->getNumElements();
3922 if (NumDstElts != SrcTy->getNumElements()) {
3923 assert(NumDstElts == 4 && "Unexpected vector size");
3924 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3925 }
3926 Rep = Builder.CreateBitCast(
3927 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3928 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3929 if (CI->arg_size() >= 3)
3930 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3931 CI->getArgOperand(1));
3932 } else if (Name.starts_with("avx512.mask.load")) {
3933 // "avx512.mask.loadu." or "avx512.mask.load."
3934 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3935 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3936 CI->getArgOperand(2), Aligned);
3937 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3938 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3939 auto *PtrTy = CI->getOperand(0)->getType();
3940 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3941 ResultTy->getNumElements());
3942 Rep = Builder.CreateIntrinsic(
3943 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3944 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3945 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3946 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3947 auto *PtrTy = CI->getArgOperand(0)->getType();
3948 Value *MaskVec =
3949 getX86MaskVec(Builder, CI->getArgOperand(2),
3950 cast<FixedVectorType>(ResultTy)->getNumElements());
3951 Rep = Builder.CreateIntrinsic(
3952 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3953 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3954 } else if (Name.starts_with("avx512.mask.compress.") ||
3955 Name.starts_with("avx512.mask.expand.")) {
3956 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3957
3958 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3959 ResultTy->getNumElements());
3960
3961 bool IsCompress = Name[12] == 'c';
3962 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3963 : Intrinsic::x86_avx512_mask_expand;
3964 Rep = Builder.CreateIntrinsic(
3965 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3966 } else if (Name.starts_with("xop.vpcom")) {
3967 bool IsSigned;
3968 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3969 Name.ends_with("uq"))
3970 IsSigned = false;
3971 else if (Name.ends_with("b") || Name.ends_with("w") ||
3972 Name.ends_with("d") || Name.ends_with("q"))
3973 IsSigned = true;
3974 else
3975 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3976
3977 unsigned Imm;
3978 if (CI->arg_size() == 3) {
3979 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3980 } else {
3981 Name = Name.substr(9); // strip off "xop.vpcom"
3982 if (Name.starts_with("lt"))
3983 Imm = 0;
3984 else if (Name.starts_with("le"))
3985 Imm = 1;
3986 else if (Name.starts_with("gt"))
3987 Imm = 2;
3988 else if (Name.starts_with("ge"))
3989 Imm = 3;
3990 else if (Name.starts_with("eq"))
3991 Imm = 4;
3992 else if (Name.starts_with("ne"))
3993 Imm = 5;
3994 else if (Name.starts_with("false"))
3995 Imm = 6;
3996 else if (Name.starts_with("true"))
3997 Imm = 7;
3998 else
3999 llvm_unreachable("Unknown condition");
4000 }
4001
4002 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
4003 } else if (Name.starts_with("xop.vpcmov")) {
4004 Value *Sel = CI->getArgOperand(2);
4005 Value *NotSel = Builder.CreateNot(Sel);
4006 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
4007 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
4008 Rep = Builder.CreateOr(Sel0, Sel1);
4009 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
4010 Name.starts_with("avx512.mask.prol")) {
4011 Rep = upgradeX86Rotate(Builder, *CI, false);
4012 } else if (Name.starts_with("avx512.pror") ||
4013 Name.starts_with("avx512.mask.pror")) {
4014 Rep = upgradeX86Rotate(Builder, *CI, true);
4015 } else if (Name.starts_with("avx512.vpshld.") ||
4016 Name.starts_with("avx512.mask.vpshld") ||
4017 Name.starts_with("avx512.maskz.vpshld")) {
4018 bool ZeroMask = Name[11] == 'z';
4019 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
4020 } else if (Name.starts_with("avx512.vpshrd.") ||
4021 Name.starts_with("avx512.mask.vpshrd") ||
4022 Name.starts_with("avx512.maskz.vpshrd")) {
4023 bool ZeroMask = Name[11] == 'z';
4024 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
4025 } else if (Name == "sse42.crc32.64.8") {
4026 Value *Trunc0 =
4027 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
4028 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
4029 {Trunc0, CI->getArgOperand(1)});
4030 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
4031 } else if (Name.starts_with("avx.vbroadcast.s") ||
4032 Name.starts_with("avx512.vbroadcast.s")) {
4033 // Replace broadcasts with a series of insertelements.
4034 auto *VecTy = cast<FixedVectorType>(CI->getType());
4035 Type *EltTy = VecTy->getElementType();
4036 unsigned EltNum = VecTy->getNumElements();
4037 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
4038 Type *I32Ty = Type::getInt32Ty(C);
4039 Rep = PoisonValue::get(VecTy);
4040 for (unsigned I = 0; I < EltNum; ++I)
4041 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
4042 } else if (Name.starts_with("sse41.pmovsx") ||
4043 Name.starts_with("sse41.pmovzx") ||
4044 Name.starts_with("avx2.pmovsx") ||
4045 Name.starts_with("avx2.pmovzx") ||
4046 Name.starts_with("avx512.mask.pmovsx") ||
4047 Name.starts_with("avx512.mask.pmovzx")) {
4048 auto *DstTy = cast<FixedVectorType>(CI->getType());
4049 unsigned NumDstElts = DstTy->getNumElements();
4050
4051 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4052 SmallVector<int, 8> ShuffleMask(NumDstElts);
4053 for (unsigned i = 0; i != NumDstElts; ++i)
4054 ShuffleMask[i] = i;
4055
4056 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
4057
4058 bool DoSext = Name.contains("pmovsx");
4059 Rep =
4060 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4061 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4062 if (CI->arg_size() == 3)
4063 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4064 CI->getArgOperand(1));
4065 } else if (Name == "avx512.mask.pmov.qd.256" ||
4066 Name == "avx512.mask.pmov.qd.512" ||
4067 Name == "avx512.mask.pmov.wb.256" ||
4068 Name == "avx512.mask.pmov.wb.512") {
4069 Type *Ty = CI->getArgOperand(1)->getType();
4070 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
4071 Rep =
4072 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4073 } else if (Name.starts_with("avx.vbroadcastf128") ||
4074 Name == "avx2.vbroadcasti128") {
4075 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4076 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4077 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4078 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4079 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4080 if (NumSrcElts == 2)
4081 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4082 else
4083 Rep = Builder.CreateShuffleVector(Load,
4084 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4085 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4086 Name.starts_with("avx512.mask.shuf.f")) {
4087 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4088 Type *VT = CI->getType();
4089 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4090 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4091 unsigned ControlBitsMask = NumLanes - 1;
4092 unsigned NumControlBits = NumLanes / 2;
4093 SmallVector<int, 8> ShuffleMask(0);
4094
4095 for (unsigned l = 0; l != NumLanes; ++l) {
4096 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4097 // We actually need the other source.
4098 if (l >= NumLanes / 2)
4099 LaneMask += NumLanes;
4100 for (unsigned i = 0; i != NumElementsInLane; ++i)
4101 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4102 }
4103 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4104 CI->getArgOperand(1), ShuffleMask);
4105 Rep =
4106 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4107 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4108 Name.starts_with("avx512.mask.broadcasti")) {
4109 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4110 ->getNumElements();
4111 unsigned NumDstElts =
4112 cast<FixedVectorType>(CI->getType())->getNumElements();
4113
4114 SmallVector<int, 8> ShuffleMask(NumDstElts);
4115 for (unsigned i = 0; i != NumDstElts; ++i)
4116 ShuffleMask[i] = i % NumSrcElts;
4117
4118 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4119 CI->getArgOperand(0), ShuffleMask);
4120 Rep =
4121 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4122 } else if (Name.starts_with("avx2.pbroadcast") ||
4123 Name.starts_with("avx2.vbroadcast") ||
4124 Name.starts_with("avx512.pbroadcast") ||
4125 Name.starts_with("avx512.mask.broadcast.s")) {
4126 // Replace vp?broadcasts with a vector shuffle.
4127 Value *Op = CI->getArgOperand(0);
4128 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4129 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4130 SmallVector<int, 8> M;
4131 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4132 Rep = Builder.CreateShuffleVector(Op, M);
4133
4134 if (CI->arg_size() == 3)
4135 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4136 CI->getArgOperand(1));
4137 } else if (Name.starts_with("sse2.padds.") ||
4138 Name.starts_with("avx2.padds.") ||
4139 Name.starts_with("avx512.padds.") ||
4140 Name.starts_with("avx512.mask.padds.")) {
4141 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4142 } else if (Name.starts_with("sse2.psubs.") ||
4143 Name.starts_with("avx2.psubs.") ||
4144 Name.starts_with("avx512.psubs.") ||
4145 Name.starts_with("avx512.mask.psubs.")) {
4146 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4147 } else if (Name.starts_with("sse2.paddus.") ||
4148 Name.starts_with("avx2.paddus.") ||
4149 Name.starts_with("avx512.mask.paddus.")) {
4150 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4151 } else if (Name.starts_with("sse2.psubus.") ||
4152 Name.starts_with("avx2.psubus.") ||
4153 Name.starts_with("avx512.mask.psubus.")) {
4154 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4155 } else if (Name.starts_with("avx512.mask.palignr.")) {
4156 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4157 CI->getArgOperand(1), CI->getArgOperand(2),
4158 CI->getArgOperand(3), CI->getArgOperand(4),
4159 false);
4160 } else if (Name.starts_with("avx512.mask.valign.")) {
4161 Rep = upgradeX86ALIGNIntrinsics(
4162 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4163 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4164 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4165 // 128/256-bit shift left specified in bits.
4166 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4167 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4168 Shift / 8); // Shift is in bits.
4169 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4170 // 128/256-bit shift right specified in bits.
4171 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4172 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4173 Shift / 8); // Shift is in bits.
4174 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4175 Name == "avx512.psll.dq.512") {
4176 // 128/256/512-bit shift left specified in bytes.
4177 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4178 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4179 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4180 Name == "avx512.psrl.dq.512") {
4181 // 128/256/512-bit shift right specified in bytes.
4182 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4183 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4184 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4185 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4186 Name.starts_with("avx2.pblendd.")) {
4187 Value *Op0 = CI->getArgOperand(0);
4188 Value *Op1 = CI->getArgOperand(1);
4189 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4190 auto *VecTy = cast<FixedVectorType>(CI->getType());
4191 unsigned NumElts = VecTy->getNumElements();
4192
4193 SmallVector<int, 16> Idxs(NumElts);
4194 for (unsigned i = 0; i != NumElts; ++i)
4195 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4196
4197 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4198 } else if (Name.starts_with("avx.vinsertf128.") ||
4199 Name == "avx2.vinserti128" ||
4200 Name.starts_with("avx512.mask.insert")) {
4201 Value *Op0 = CI->getArgOperand(0);
4202 Value *Op1 = CI->getArgOperand(1);
4203 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4204 unsigned DstNumElts =
4205 cast<FixedVectorType>(CI->getType())->getNumElements();
4206 unsigned SrcNumElts =
4207 cast<FixedVectorType>(Op1->getType())->getNumElements();
4208 unsigned Scale = DstNumElts / SrcNumElts;
4209
4210 // Mask off the high bits of the immediate value; hardware ignores those.
4211 Imm = Imm % Scale;
4212
4213 // Extend the second operand into a vector the size of the destination.
4214 SmallVector<int, 8> Idxs(DstNumElts);
4215 for (unsigned i = 0; i != SrcNumElts; ++i)
4216 Idxs[i] = i;
4217 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4218 Idxs[i] = SrcNumElts;
4219 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4220
4221 // Insert the second operand into the first operand.
4222
4223 // Note that there is no guarantee that instruction lowering will actually
4224 // produce a vinsertf128 instruction for the created shuffles. In
4225 // particular, the 0 immediate case involves no lane changes, so it can
4226 // be handled as a blend.
4227
4228 // Example of shuffle mask for 32-bit elements:
4229 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4230 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4231
4232 // First fill with identify mask.
4233 for (unsigned i = 0; i != DstNumElts; ++i)
4234 Idxs[i] = i;
4235 // Then replace the elements where we need to insert.
4236 for (unsigned i = 0; i != SrcNumElts; ++i)
4237 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4238 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4239
4240 // If the intrinsic has a mask operand, handle that.
4241 if (CI->arg_size() == 5)
4242 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4243 CI->getArgOperand(3));
4244 } else if (Name.starts_with("avx.vextractf128.") ||
4245 Name == "avx2.vextracti128" ||
4246 Name.starts_with("avx512.mask.vextract")) {
4247 Value *Op0 = CI->getArgOperand(0);
4248 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4249 unsigned DstNumElts =
4250 cast<FixedVectorType>(CI->getType())->getNumElements();
4251 unsigned SrcNumElts =
4252 cast<FixedVectorType>(Op0->getType())->getNumElements();
4253 unsigned Scale = SrcNumElts / DstNumElts;
4254
4255 // Mask off the high bits of the immediate value; hardware ignores those.
4256 Imm = Imm % Scale;
4257
4258 // Get indexes for the subvector of the input vector.
4259 SmallVector<int, 8> Idxs(DstNumElts);
4260 for (unsigned i = 0; i != DstNumElts; ++i) {
4261 Idxs[i] = i + (Imm * DstNumElts);
4262 }
4263 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4264
4265 // If the intrinsic has a mask operand, handle that.
4266 if (CI->arg_size() == 4)
4267 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4268 CI->getArgOperand(2));
4269 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4270 Name.starts_with("avx512.mask.perm.di.")) {
4271 Value *Op0 = CI->getArgOperand(0);
4272 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4273 auto *VecTy = cast<FixedVectorType>(CI->getType());
4274 unsigned NumElts = VecTy->getNumElements();
4275
4276 SmallVector<int, 8> Idxs(NumElts);
4277 for (unsigned i = 0; i != NumElts; ++i)
4278 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4279
4280 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4281
4282 if (CI->arg_size() == 4)
4283 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4284 CI->getArgOperand(2));
4285 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4286 // The immediate permute control byte looks like this:
4287 // [1:0] - select 128 bits from sources for low half of destination
4288 // [2] - ignore
4289 // [3] - zero low half of destination
4290 // [5:4] - select 128 bits from sources for high half of destination
4291 // [6] - ignore
4292 // [7] - zero high half of destination
4293
4294 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4295
4296 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4297 unsigned HalfSize = NumElts / 2;
4298 SmallVector<int, 8> ShuffleMask(NumElts);
4299
4300 // Determine which operand(s) are actually in use for this instruction.
4301 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4302 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4303
4304 // If needed, replace operands based on zero mask.
4305 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4306 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4307
4308 // Permute low half of result.
4309 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4310 for (unsigned i = 0; i < HalfSize; ++i)
4311 ShuffleMask[i] = StartIndex + i;
4312
4313 // Permute high half of result.
4314 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4315 for (unsigned i = 0; i < HalfSize; ++i)
4316 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4317
4318 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4319
4320 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4321 Name.starts_with("avx512.mask.vpermil.p") ||
4322 Name.starts_with("avx512.mask.pshuf.d.")) {
4323 Value *Op0 = CI->getArgOperand(0);
4324 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4325 auto *VecTy = cast<FixedVectorType>(CI->getType());
4326 unsigned NumElts = VecTy->getNumElements();
4327 // Calculate the size of each index in the immediate.
4328 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4329 unsigned IdxMask = ((1 << IdxSize) - 1);
4330
4331 SmallVector<int, 8> Idxs(NumElts);
4332 // Lookup the bits for this element, wrapping around the immediate every
4333 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4334 // to offset by the first index of each group.
4335 for (unsigned i = 0; i != NumElts; ++i)
4336 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4337
4338 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4339
4340 if (CI->arg_size() == 4)
4341 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4342 CI->getArgOperand(2));
4343 } else if (Name == "sse2.pshufl.w" ||
4344 Name.starts_with("avx512.mask.pshufl.w.")) {
4345 Value *Op0 = CI->getArgOperand(0);
4346 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4347 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4348
4349 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4350 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4351
4352 SmallVector<int, 16> Idxs(NumElts);
4353 for (unsigned l = 0; l != NumElts; l += 8) {
4354 for (unsigned i = 0; i != 4; ++i)
4355 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4356 for (unsigned i = 4; i != 8; ++i)
4357 Idxs[i + l] = i + l;
4358 }
4359
4360 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4361
4362 if (CI->arg_size() == 4)
4363 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4364 CI->getArgOperand(2));
4365 } else if (Name == "sse2.pshufh.w" ||
4366 Name.starts_with("avx512.mask.pshufh.w.")) {
4367 Value *Op0 = CI->getArgOperand(0);
4368 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4369 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4370
4371 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4372 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4373
4374 SmallVector<int, 16> Idxs(NumElts);
4375 for (unsigned l = 0; l != NumElts; l += 8) {
4376 for (unsigned i = 0; i != 4; ++i)
4377 Idxs[i + l] = i + l;
4378 for (unsigned i = 0; i != 4; ++i)
4379 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4380 }
4381
4382 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4383
4384 if (CI->arg_size() == 4)
4385 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4386 CI->getArgOperand(2));
4387 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4388 Value *Op0 = CI->getArgOperand(0);
4389 Value *Op1 = CI->getArgOperand(1);
4390 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4391 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4392
4393 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4394 unsigned HalfLaneElts = NumLaneElts / 2;
4395
4396 SmallVector<int, 16> Idxs(NumElts);
4397 for (unsigned i = 0; i != NumElts; ++i) {
4398 // Base index is the starting element of the lane.
4399 Idxs[i] = i - (i % NumLaneElts);
4400 // If we are half way through the lane switch to the other source.
4401 if ((i % NumLaneElts) >= HalfLaneElts)
4402 Idxs[i] += NumElts;
4403 // Now select the specific element. By adding HalfLaneElts bits from
4404 // the immediate. Wrapping around the immediate every 8-bits.
4405 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4406 }
4407
4408 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4409
4410 Rep =
4411 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4412 } else if (Name.starts_with("avx512.mask.movddup") ||
4413 Name.starts_with("avx512.mask.movshdup") ||
4414 Name.starts_with("avx512.mask.movsldup")) {
4415 Value *Op0 = CI->getArgOperand(0);
4416 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4417 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4418
4419 unsigned Offset = 0;
4420 if (Name.starts_with("avx512.mask.movshdup."))
4421 Offset = 1;
4422
4423 SmallVector<int, 16> Idxs(NumElts);
4424 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4425 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4426 Idxs[i + l + 0] = i + l + Offset;
4427 Idxs[i + l + 1] = i + l + Offset;
4428 }
4429
4430 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4431
4432 Rep =
4433 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4434 } else if (Name.starts_with("avx512.mask.punpckl") ||
4435 Name.starts_with("avx512.mask.unpckl.")) {
4436 Value *Op0 = CI->getArgOperand(0);
4437 Value *Op1 = CI->getArgOperand(1);
4438 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4439 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4440
4441 SmallVector<int, 64> Idxs(NumElts);
4442 for (int l = 0; l != NumElts; l += NumLaneElts)
4443 for (int i = 0; i != NumLaneElts; ++i)
4444 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4445
4446 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4447
4448 Rep =
4449 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4450 } else if (Name.starts_with("avx512.mask.punpckh") ||
4451 Name.starts_with("avx512.mask.unpckh.")) {
4452 Value *Op0 = CI->getArgOperand(0);
4453 Value *Op1 = CI->getArgOperand(1);
4454 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4455 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4456
4457 SmallVector<int, 64> Idxs(NumElts);
4458 for (int l = 0; l != NumElts; l += NumLaneElts)
4459 for (int i = 0; i != NumLaneElts; ++i)
4460 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4461
4462 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4463
4464 Rep =
4465 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4466 } else if (Name.starts_with("avx512.mask.and.") ||
4467 Name.starts_with("avx512.mask.pand.")) {
4468 VectorType *FTy = cast<VectorType>(CI->getType());
4469 VectorType *ITy = VectorType::getInteger(FTy);
4470 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4471 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4472 Rep = Builder.CreateBitCast(Rep, FTy);
4473 Rep =
4474 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4475 } else if (Name.starts_with("avx512.mask.andn.") ||
4476 Name.starts_with("avx512.mask.pandn.")) {
4477 VectorType *FTy = cast<VectorType>(CI->getType());
4478 VectorType *ITy = VectorType::getInteger(FTy);
4479 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4480 Rep = Builder.CreateAnd(Rep,
4481 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4482 Rep = Builder.CreateBitCast(Rep, FTy);
4483 Rep =
4484 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4485 } else if (Name.starts_with("avx512.mask.or.") ||
4486 Name.starts_with("avx512.mask.por.")) {
4487 VectorType *FTy = cast<VectorType>(CI->getType());
4488 VectorType *ITy = VectorType::getInteger(FTy);
4489 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4490 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4491 Rep = Builder.CreateBitCast(Rep, FTy);
4492 Rep =
4493 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4494 } else if (Name.starts_with("avx512.mask.xor.") ||
4495 Name.starts_with("avx512.mask.pxor.")) {
4496 VectorType *FTy = cast<VectorType>(CI->getType());
4497 VectorType *ITy = VectorType::getInteger(FTy);
4498 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4499 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4500 Rep = Builder.CreateBitCast(Rep, FTy);
4501 Rep =
4502 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4503 } else if (Name.starts_with("avx512.mask.padd.")) {
4504 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4505 Rep =
4506 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4507 } else if (Name.starts_with("avx512.mask.psub.")) {
4508 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4509 Rep =
4510 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4511 } else if (Name.starts_with("avx512.mask.pmull.")) {
4512 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4513 Rep =
4514 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4515 } else if (Name.starts_with("avx512.mask.add.p")) {
4516 if (Name.ends_with(".512")) {
4517 Intrinsic::ID IID;
4518 if (Name[17] == 's')
4519 IID = Intrinsic::x86_avx512_add_ps_512;
4520 else
4521 IID = Intrinsic::x86_avx512_add_pd_512;
4522
4523 Rep = Builder.CreateIntrinsic(
4524 IID,
4525 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4526 } else {
4527 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4528 }
4529 Rep =
4530 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4531 } else if (Name.starts_with("avx512.mask.div.p")) {
4532 if (Name.ends_with(".512")) {
4533 Intrinsic::ID IID;
4534 if (Name[17] == 's')
4535 IID = Intrinsic::x86_avx512_div_ps_512;
4536 else
4537 IID = Intrinsic::x86_avx512_div_pd_512;
4538
4539 Rep = Builder.CreateIntrinsic(
4540 IID,
4541 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4542 } else {
4543 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4544 }
4545 Rep =
4546 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4547 } else if (Name.starts_with("avx512.mask.mul.p")) {
4548 if (Name.ends_with(".512")) {
4549 Intrinsic::ID IID;
4550 if (Name[17] == 's')
4551 IID = Intrinsic::x86_avx512_mul_ps_512;
4552 else
4553 IID = Intrinsic::x86_avx512_mul_pd_512;
4554
4555 Rep = Builder.CreateIntrinsic(
4556 IID,
4557 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4558 } else {
4559 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4560 }
4561 Rep =
4562 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4563 } else if (Name.starts_with("avx512.mask.sub.p")) {
4564 if (Name.ends_with(".512")) {
4565 Intrinsic::ID IID;
4566 if (Name[17] == 's')
4567 IID = Intrinsic::x86_avx512_sub_ps_512;
4568 else
4569 IID = Intrinsic::x86_avx512_sub_pd_512;
4570
4571 Rep = Builder.CreateIntrinsic(
4572 IID,
4573 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4574 } else {
4575 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4576 }
4577 Rep =
4578 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4579 } else if ((Name.starts_with("avx512.mask.max.p") ||
4580 Name.starts_with("avx512.mask.min.p")) &&
4581 Name.drop_front(18) == ".512") {
4582 bool IsDouble = Name[17] == 'd';
4583 bool IsMin = Name[13] == 'i';
4584 static const Intrinsic::ID MinMaxTbl[2][2] = {
4585 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4586 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4587 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4588
4589 Rep = Builder.CreateIntrinsic(
4590 IID,
4591 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4592 Rep =
4593 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4594 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4595 Rep =
4596 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4597 {CI->getArgOperand(0), Builder.getInt1(false)});
4598 Rep =
4599 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4600 } else if (Name.starts_with("avx512.mask.psll")) {
4601 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4602 bool IsVariable = Name[16] == 'v';
4603 char Size = Name[16] == '.' ? Name[17]
4604 : Name[17] == '.' ? Name[18]
4605 : Name[18] == '.' ? Name[19]
4606 : Name[20];
4607
4608 Intrinsic::ID IID;
4609 if (IsVariable && Name[17] != '.') {
4610 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4611 IID = Intrinsic::x86_avx2_psllv_q;
4612 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4613 IID = Intrinsic::x86_avx2_psllv_q_256;
4614 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4615 IID = Intrinsic::x86_avx2_psllv_d;
4616 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4617 IID = Intrinsic::x86_avx2_psllv_d_256;
4618 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4619 IID = Intrinsic::x86_avx512_psllv_w_128;
4620 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4621 IID = Intrinsic::x86_avx512_psllv_w_256;
4622 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4623 IID = Intrinsic::x86_avx512_psllv_w_512;
4624 else
4625 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4626 } else if (Name.ends_with(".128")) {
4627 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4628 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4629 : Intrinsic::x86_sse2_psll_d;
4630 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4631 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4632 : Intrinsic::x86_sse2_psll_q;
4633 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4634 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4635 : Intrinsic::x86_sse2_psll_w;
4636 else
4637 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4638 } else if (Name.ends_with(".256")) {
4639 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4640 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4641 : Intrinsic::x86_avx2_psll_d;
4642 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4643 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4644 : Intrinsic::x86_avx2_psll_q;
4645 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4646 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4647 : Intrinsic::x86_avx2_psll_w;
4648 else
4649 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4650 } else {
4651 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4652 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4653 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4654 : Intrinsic::x86_avx512_psll_d_512;
4655 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4656 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4657 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4658 : Intrinsic::x86_avx512_psll_q_512;
4659 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4660 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4661 : Intrinsic::x86_avx512_psll_w_512;
4662 else
4663 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4664 }
4665
4666 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4667 } else if (Name.starts_with("avx512.mask.psrl")) {
4668 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4669 bool IsVariable = Name[16] == 'v';
4670 char Size = Name[16] == '.' ? Name[17]
4671 : Name[17] == '.' ? Name[18]
4672 : Name[18] == '.' ? Name[19]
4673 : Name[20];
4674
4675 Intrinsic::ID IID;
4676 if (IsVariable && Name[17] != '.') {
4677 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4678 IID = Intrinsic::x86_avx2_psrlv_q;
4679 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4680 IID = Intrinsic::x86_avx2_psrlv_q_256;
4681 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4682 IID = Intrinsic::x86_avx2_psrlv_d;
4683 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4684 IID = Intrinsic::x86_avx2_psrlv_d_256;
4685 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4686 IID = Intrinsic::x86_avx512_psrlv_w_128;
4687 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4688 IID = Intrinsic::x86_avx512_psrlv_w_256;
4689 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4690 IID = Intrinsic::x86_avx512_psrlv_w_512;
4691 else
4692 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4693 } else if (Name.ends_with(".128")) {
4694 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4695 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4696 : Intrinsic::x86_sse2_psrl_d;
4697 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4698 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4699 : Intrinsic::x86_sse2_psrl_q;
4700 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4701 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4702 : Intrinsic::x86_sse2_psrl_w;
4703 else
4704 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4705 } else if (Name.ends_with(".256")) {
4706 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4707 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4708 : Intrinsic::x86_avx2_psrl_d;
4709 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4710 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4711 : Intrinsic::x86_avx2_psrl_q;
4712 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4713 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4714 : Intrinsic::x86_avx2_psrl_w;
4715 else
4716 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4717 } else {
4718 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4719 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4720 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4721 : Intrinsic::x86_avx512_psrl_d_512;
4722 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4723 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4724 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4725 : Intrinsic::x86_avx512_psrl_q_512;
4726 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4727 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4728 : Intrinsic::x86_avx512_psrl_w_512;
4729 else
4730 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4731 }
4732
4733 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4734 } else if (Name.starts_with("avx512.mask.psra")) {
4735 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4736 bool IsVariable = Name[16] == 'v';
4737 char Size = Name[16] == '.' ? Name[17]
4738 : Name[17] == '.' ? Name[18]
4739 : Name[18] == '.' ? Name[19]
4740 : Name[20];
4741
4742 Intrinsic::ID IID;
4743 if (IsVariable && Name[17] != '.') {
4744 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4745 IID = Intrinsic::x86_avx2_psrav_d;
4746 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4747 IID = Intrinsic::x86_avx2_psrav_d_256;
4748 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4749 IID = Intrinsic::x86_avx512_psrav_w_128;
4750 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4751 IID = Intrinsic::x86_avx512_psrav_w_256;
4752 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4753 IID = Intrinsic::x86_avx512_psrav_w_512;
4754 else
4755 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4756 } else if (Name.ends_with(".128")) {
4757 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4758 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4759 : Intrinsic::x86_sse2_psra_d;
4760 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4761 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4762 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4763 : Intrinsic::x86_avx512_psra_q_128;
4764 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4765 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4766 : Intrinsic::x86_sse2_psra_w;
4767 else
4768 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4769 } else if (Name.ends_with(".256")) {
4770 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4771 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4772 : Intrinsic::x86_avx2_psra_d;
4773 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4774 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4775 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4776 : Intrinsic::x86_avx512_psra_q_256;
4777 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4778 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4779 : Intrinsic::x86_avx2_psra_w;
4780 else
4781 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4782 } else {
4783 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4784 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4785 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4786 : Intrinsic::x86_avx512_psra_d_512;
4787 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4788 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4789 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4790 : Intrinsic::x86_avx512_psra_q_512;
4791 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4792 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4793 : Intrinsic::x86_avx512_psra_w_512;
4794 else
4795 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4796 }
4797
4798 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4799 } else if (Name.starts_with("avx512.mask.move.s")) {
4800 Rep = upgradeMaskedMove(Builder, *CI);
4801 } else if (Name.starts_with("avx512.cvtmask2")) {
4802 Rep = upgradeMaskToInt(Builder, *CI);
4803 } else if (Name.ends_with(".movntdqa")) {
4804 MDNode *Node = MDNode::get(
4805 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4806
4807 LoadInst *LI = Builder.CreateAlignedLoad(
4808 CI->getType(), CI->getArgOperand(0),
4809 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4810 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4811 Rep = LI;
4812 } else if (Name.starts_with("fma.vfmadd.") ||
4813 Name.starts_with("fma.vfmsub.") ||
4814 Name.starts_with("fma.vfnmadd.") ||
4815 Name.starts_with("fma.vfnmsub.")) {
4816 bool NegMul = Name[6] == 'n';
4817 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4818 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4819
4820 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4821 CI->getArgOperand(2)};
4822
4823 if (IsScalar) {
4824 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4825 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4826 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4827 }
4828
4829 if (NegMul && !IsScalar)
4830 Ops[0] = Builder.CreateFNeg(Ops[0]);
4831 if (NegMul && IsScalar)
4832 Ops[1] = Builder.CreateFNeg(Ops[1]);
4833 if (NegAcc)
4834 Ops[2] = Builder.CreateFNeg(Ops[2]);
4835
4836 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4837
4838 if (IsScalar)
4839 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4840 } else if (Name.starts_with("fma4.vfmadd.s")) {
4841 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4842 CI->getArgOperand(2)};
4843
4844 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4845 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4846 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4847
4848 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4849
4850 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4851 Rep, (uint64_t)0);
4852 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4853 Name.starts_with("avx512.maskz.vfmadd.s") ||
4854 Name.starts_with("avx512.mask3.vfmadd.s") ||
4855 Name.starts_with("avx512.mask3.vfmsub.s") ||
4856 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4857 bool IsMask3 = Name[11] == '3';
4858 bool IsMaskZ = Name[11] == 'z';
4859 // Drop the "avx512.mask." to make it easier.
4860 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4861 bool NegMul = Name[2] == 'n';
4862 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4863
4864 Value *A = CI->getArgOperand(0);
4865 Value *B = CI->getArgOperand(1);
4866 Value *C = CI->getArgOperand(2);
4867
4868 if (NegMul && (IsMask3 || IsMaskZ))
4869 A = Builder.CreateFNeg(A);
4870 if (NegMul && !(IsMask3 || IsMaskZ))
4871 B = Builder.CreateFNeg(B);
4872 if (NegAcc)
4873 C = Builder.CreateFNeg(C);
4874
4875 A = Builder.CreateExtractElement(A, (uint64_t)0);
4876 B = Builder.CreateExtractElement(B, (uint64_t)0);
4877 C = Builder.CreateExtractElement(C, (uint64_t)0);
4878
4879 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4880 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4881 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4882
4883 Intrinsic::ID IID;
4884 if (Name.back() == 'd')
4885 IID = Intrinsic::x86_avx512_vfmadd_f64;
4886 else
4887 IID = Intrinsic::x86_avx512_vfmadd_f32;
4888 Rep = Builder.CreateIntrinsic(IID, Ops);
4889 } else {
4890 Rep = Builder.CreateFMA(A, B, C);
4891 }
4892
4893 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4894 : IsMask3 ? C
4895 : A;
4896
4897 // For Mask3 with NegAcc, we need to create a new extractelement that
4898 // avoids the negation above.
4899 if (NegAcc && IsMask3)
4900 PassThru =
4901 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4902
4903 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4904 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4905 (uint64_t)0);
4906 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4907 Name.starts_with("avx512.mask.vfnmadd.p") ||
4908 Name.starts_with("avx512.mask.vfnmsub.p") ||
4909 Name.starts_with("avx512.mask3.vfmadd.p") ||
4910 Name.starts_with("avx512.mask3.vfmsub.p") ||
4911 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4912 Name.starts_with("avx512.maskz.vfmadd.p")) {
4913 bool IsMask3 = Name[11] == '3';
4914 bool IsMaskZ = Name[11] == 'z';
4915 // Drop the "avx512.mask." to make it easier.
4916 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4917 bool NegMul = Name[2] == 'n';
4918 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4919
4920 Value *A = CI->getArgOperand(0);
4921 Value *B = CI->getArgOperand(1);
4922 Value *C = CI->getArgOperand(2);
4923
4924 if (NegMul && (IsMask3 || IsMaskZ))
4925 A = Builder.CreateFNeg(A);
4926 if (NegMul && !(IsMask3 || IsMaskZ))
4927 B = Builder.CreateFNeg(B);
4928 if (NegAcc)
4929 C = Builder.CreateFNeg(C);
4930
4931 if (CI->arg_size() == 5 &&
4932 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4933 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4934 Intrinsic::ID IID;
4935 // Check the character before ".512" in string.
4936 if (Name[Name.size() - 5] == 's')
4937 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4938 else
4939 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4940
4941 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4942 } else {
4943 Rep = Builder.CreateFMA(A, B, C);
4944 }
4945
4946 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4947 : IsMask3 ? CI->getArgOperand(2)
4948 : CI->getArgOperand(0);
4949
4950 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4951 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4952 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4953 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4954 Intrinsic::ID IID;
4955 if (VecWidth == 128 && EltWidth == 32)
4956 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4957 else if (VecWidth == 256 && EltWidth == 32)
4958 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4959 else if (VecWidth == 128 && EltWidth == 64)
4960 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4961 else if (VecWidth == 256 && EltWidth == 64)
4962 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4963 else
4964 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4965
4966 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4967 CI->getArgOperand(2)};
4968 Ops[2] = Builder.CreateFNeg(Ops[2]);
4969 Rep = Builder.CreateIntrinsic(IID, Ops);
4970 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4971 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4972 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4973 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4974 bool IsMask3 = Name[11] == '3';
4975 bool IsMaskZ = Name[11] == 'z';
4976 // Drop the "avx512.mask." to make it easier.
4977 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4978 bool IsSubAdd = Name[3] == 's';
4979 if (CI->arg_size() == 5) {
4980 Intrinsic::ID IID;
4981 // Check the character before ".512" in string.
4982 if (Name[Name.size() - 5] == 's')
4983 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4984 else
4985 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4986
4987 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4988 CI->getArgOperand(2), CI->getArgOperand(4)};
4989 if (IsSubAdd)
4990 Ops[2] = Builder.CreateFNeg(Ops[2]);
4991
4992 Rep = Builder.CreateIntrinsic(IID, Ops);
4993 } else {
4994 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4995
4996 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4997 CI->getArgOperand(2)};
4998
4999 Function *FMA = Intrinsic::getOrInsertDeclaration(
5000 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
5001 Value *Odd = Builder.CreateCall(FMA, Ops);
5002 Ops[2] = Builder.CreateFNeg(Ops[2]);
5003 Value *Even = Builder.CreateCall(FMA, Ops);
5004
5005 if (IsSubAdd)
5006 std::swap(Even, Odd);
5007
5008 SmallVector<int, 32> Idxs(NumElts);
5009 for (int i = 0; i != NumElts; ++i)
5010 Idxs[i] = i + (i % 2) * NumElts;
5011
5012 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
5013 }
5014
5015 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
5016 : IsMask3 ? CI->getArgOperand(2)
5017 : CI->getArgOperand(0);
5018
5019 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5020 } else if (Name.starts_with("avx512.mask.pternlog.") ||
5021 Name.starts_with("avx512.maskz.pternlog.")) {
5022 bool ZeroMask = Name[11] == 'z';
5023 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5024 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
5025 Intrinsic::ID IID;
5026 if (VecWidth == 128 && EltWidth == 32)
5027 IID = Intrinsic::x86_avx512_pternlog_d_128;
5028 else if (VecWidth == 256 && EltWidth == 32)
5029 IID = Intrinsic::x86_avx512_pternlog_d_256;
5030 else if (VecWidth == 512 && EltWidth == 32)
5031 IID = Intrinsic::x86_avx512_pternlog_d_512;
5032 else if (VecWidth == 128 && EltWidth == 64)
5033 IID = Intrinsic::x86_avx512_pternlog_q_128;
5034 else if (VecWidth == 256 && EltWidth == 64)
5035 IID = Intrinsic::x86_avx512_pternlog_q_256;
5036 else if (VecWidth == 512 && EltWidth == 64)
5037 IID = Intrinsic::x86_avx512_pternlog_q_512;
5038 else
5039 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5040
5041 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5042 CI->getArgOperand(2), CI->getArgOperand(3)};
5043 Rep = Builder.CreateIntrinsic(IID, Args);
5044 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5045 : CI->getArgOperand(0);
5046 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
5047 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
5048 Name.starts_with("avx512.maskz.vpmadd52")) {
5049 bool ZeroMask = Name[11] == 'z';
5050 bool High = Name[20] == 'h' || Name[21] == 'h';
5051 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5052 Intrinsic::ID IID;
5053 if (VecWidth == 128 && !High)
5054 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5055 else if (VecWidth == 256 && !High)
5056 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5057 else if (VecWidth == 512 && !High)
5058 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5059 else if (VecWidth == 128 && High)
5060 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5061 else if (VecWidth == 256 && High)
5062 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5063 else if (VecWidth == 512 && High)
5064 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5065 else
5066 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5067
5068 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5069 CI->getArgOperand(2)};
5070 Rep = Builder.CreateIntrinsic(IID, Args);
5071 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5072 : CI->getArgOperand(0);
5073 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5074 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5075 Name.starts_with("avx512.mask.vpermt2var.") ||
5076 Name.starts_with("avx512.maskz.vpermt2var.")) {
5077 bool ZeroMask = Name[11] == 'z';
5078 bool IndexForm = Name[17] == 'i';
5079 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5080 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5081 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5082 Name.starts_with("avx512.mask.vpdpbusds.") ||
5083 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5084 bool ZeroMask = Name[11] == 'z';
5085 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5086 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5087 Intrinsic::ID IID;
5088 if (VecWidth == 128 && !IsSaturating)
5089 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5090 else if (VecWidth == 256 && !IsSaturating)
5091 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5092 else if (VecWidth == 512 && !IsSaturating)
5093 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5094 else if (VecWidth == 128 && IsSaturating)
5095 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5096 else if (VecWidth == 256 && IsSaturating)
5097 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5098 else if (VecWidth == 512 && IsSaturating)
5099 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5100 else
5101 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5102
5103 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5104 CI->getArgOperand(2)};
5105
5106 // Input arguments types were incorrectly set to vectors of i32 before but
5107 // they should be vectors of i8. Insert bit cast when encountering the old
5108 // types
5109 if (Args[1]->getType()->isVectorTy() &&
5110 cast<VectorType>(Args[1]->getType())
5111 ->getElementType()
5112 ->isIntegerTy(32) &&
5113 Args[2]->getType()->isVectorTy() &&
5114 cast<VectorType>(Args[2]->getType())
5115 ->getElementType()
5116 ->isIntegerTy(32)) {
5117 Type *NewArgType = nullptr;
5118 if (VecWidth == 128)
5119 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5120 else if (VecWidth == 256)
5121 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5122 else if (VecWidth == 512)
5123 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5124 else
5125 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5126 CI);
5127
5128 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5129 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5130 }
5131
5132 Rep = Builder.CreateIntrinsic(IID, Args);
5133 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5134 : CI->getArgOperand(0);
5135 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5136 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5137 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5138 Name.starts_with("avx512.mask.vpdpwssds.") ||
5139 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5140 bool ZeroMask = Name[11] == 'z';
5141 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5142 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5143 Intrinsic::ID IID;
5144 if (VecWidth == 128 && !IsSaturating)
5145 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5146 else if (VecWidth == 256 && !IsSaturating)
5147 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5148 else if (VecWidth == 512 && !IsSaturating)
5149 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5150 else if (VecWidth == 128 && IsSaturating)
5151 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5152 else if (VecWidth == 256 && IsSaturating)
5153 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5154 else if (VecWidth == 512 && IsSaturating)
5155 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5156 else
5157 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5158
5159 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5160 CI->getArgOperand(2)};
5161
5162 // Input arguments types were incorrectly set to vectors of i32 before but
5163 // they should be vectors of i16. Insert bit cast when encountering the old
5164 // types
5165 if (Args[1]->getType()->isVectorTy() &&
5166 cast<VectorType>(Args[1]->getType())
5167 ->getElementType()
5168 ->isIntegerTy(32) &&
5169 Args[2]->getType()->isVectorTy() &&
5170 cast<VectorType>(Args[2]->getType())
5171 ->getElementType()
5172 ->isIntegerTy(32)) {
5173 Type *NewArgType = nullptr;
5174 if (VecWidth == 128)
5175 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5176 else if (VecWidth == 256)
5177 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5178 else if (VecWidth == 512)
5179 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5180 else
5181 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5182 CI);
5183
5184 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5185 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5186 }
5187
5188 Rep = Builder.CreateIntrinsic(IID, Args);
5189 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5190 : CI->getArgOperand(0);
5191 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5192 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5193 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5194 Name == "subborrow.u32" || Name == "subborrow.u64") {
5195 Intrinsic::ID IID;
5196 if (Name[0] == 'a' && Name.back() == '2')
5197 IID = Intrinsic::x86_addcarry_32;
5198 else if (Name[0] == 'a' && Name.back() == '4')
5199 IID = Intrinsic::x86_addcarry_64;
5200 else if (Name[0] == 's' && Name.back() == '2')
5201 IID = Intrinsic::x86_subborrow_32;
5202 else if (Name[0] == 's' && Name.back() == '4')
5203 IID = Intrinsic::x86_subborrow_64;
5204 else
5205 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5206
5207 // Make a call with 3 operands.
5208 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5209 CI->getArgOperand(2)};
5210 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5211
5212 // Extract the second result and store it.
5213 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5214 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5215 // Replace the original call result with the first result of the new call.
5216 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5217
5218 CI->replaceAllUsesWith(CF);
5219 Rep = nullptr;
5220 } else if (Name.starts_with("avx512.mask.") &&
5221 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5222 // Rep will be updated by the call in the condition.
5223 } else if (Name.starts_with("bmi.pdep.")) {
5224 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5225 } else if (Name.starts_with("bmi.pext.")) {
5226 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5227 } else
5228 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5229
5230 return Rep;
5231}
5232
5233static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5234 Function *F, IRBuilder<> &Builder) {
5235 if (Name.starts_with("neon.bfcvt")) {
5236 if (Name.starts_with("neon.bfcvtn2")) {
5237 SmallVector<int, 32> LoMask(4);
5238 std::iota(LoMask.begin(), LoMask.end(), 0);
5239 SmallVector<int, 32> ConcatMask(8);
5240 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5241 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5242 Value *Trunc =
5243 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5244 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5245 } else if (Name.starts_with("neon.bfcvtn")) {
5246 SmallVector<int, 32> ConcatMask(8);
5247 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5248 Type *V4BF16 =
5249 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5250 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5251 dbgs() << "Trunc: " << *Trunc << "\n";
5252 return Builder.CreateShuffleVector(
5253 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5254 } else {
5255 return Builder.CreateFPTrunc(CI->getOperand(0),
5256 Type::getBFloatTy(F->getContext()));
5257 }
5258 } else if (Name.starts_with("sve.fcvt")) {
5259 Intrinsic::ID NewID =
5260 StringSwitch<Intrinsic::ID>(Name)
5261 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5262 .Case("sve.fcvtnt.bf16f32",
5263 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5264 .Default(Intrinsic::not_intrinsic);
5265 if (NewID == Intrinsic::not_intrinsic)
5266 llvm_unreachable("Unhandled Intrinsic!");
5267
5268 SmallVector<Value *, 3> Args(CI->args());
5269
5270 // The original intrinsics incorrectly used a predicate based on the
5271 // smallest element type rather than the largest.
5272 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5273 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5274
5275 if (Args[1]->getType() != BadPredTy)
5276 llvm_unreachable("Unexpected predicate type!");
5277
5278 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5279 BadPredTy, Args[1]);
5280 Args[1] = Builder.CreateIntrinsic(
5281 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5282
5283 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5284 CI->getName());
5285 }
5286
5287 if (Name == "neon.vcvtfp2hf")
5288 return Builder.CreateBitCast(
5289 Builder.CreateFPTrunc(
5290 CI->getOperand(0),
5291 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5292 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5293 if (Name == "neon.vcvthf2fp")
5294 return Builder.CreateFPExt(
5295 Builder.CreateBitCast(
5296 CI->getOperand(0),
5297 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5298 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5299
5300 llvm_unreachable("Unhandled Intrinsic!");
5301}
5302
5303static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5304 IRBuilder<> &Builder) {
5305 if (Name == "mve.vctp64.old") {
5306 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5307 // correct type.
5308 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5309 CI->getArgOperand(0),
5310 /*FMFSource=*/nullptr, CI->getName());
5311 Value *C1 = Builder.CreateIntrinsic(
5312 Intrinsic::arm_mve_pred_v2i,
5313 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5314 return Builder.CreateIntrinsic(
5315 Intrinsic::arm_mve_pred_i2v,
5316 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5317 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5318 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5319 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5320 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5321 Name ==
5322 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5323 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5324 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5325 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5326 Name ==
5327 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5328 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5329 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5330 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5331 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5332 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5333 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5334 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5335 std::vector<Type *> Tys;
5336 unsigned ID = CI->getIntrinsicID();
5337 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5338 switch (ID) {
5339 case Intrinsic::arm_mve_mull_int_predicated:
5340 case Intrinsic::arm_mve_vqdmull_predicated:
5341 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5342 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5343 break;
5344 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5345 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5346 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5347 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5348 V2I1Ty};
5349 break;
5350 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5351 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5352 CI->getOperand(1)->getType(), V2I1Ty};
5353 break;
5354 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5355 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5356 CI->getOperand(2)->getType(), V2I1Ty};
5357 break;
5358 case Intrinsic::arm_cde_vcx1q_predicated:
5359 case Intrinsic::arm_cde_vcx1qa_predicated:
5360 case Intrinsic::arm_cde_vcx2q_predicated:
5361 case Intrinsic::arm_cde_vcx2qa_predicated:
5362 case Intrinsic::arm_cde_vcx3q_predicated:
5363 case Intrinsic::arm_cde_vcx3qa_predicated:
5364 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5365 break;
5366 default:
5367 llvm_unreachable("Unhandled Intrinsic!");
5368 }
5369
5370 std::vector<Value *> Ops;
5371 for (Value *Op : CI->args()) {
5372 Type *Ty = Op->getType();
5373 if (Ty->getScalarSizeInBits() == 1) {
5374 Value *C1 = Builder.CreateIntrinsic(
5375 Intrinsic::arm_mve_pred_v2i,
5376 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5377 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5378 }
5379 Ops.push_back(Op);
5380 }
5381
5382 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5383 CI->getName());
5384 }
5385 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5386}
5387
5388// These are expected to have the arguments:
5389// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5390//
5391// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5392//
5393static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5394 Function *F, IRBuilder<> &Builder) {
5395 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5396 // for compatibility.
5397 auto UpgradeLegacyWMMAIUIntrinsicCall =
5398 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5399 ArrayRef<Type *> OverloadTys) -> Value * {
5400 // Prepare arguments, append clamp=0 for compatibility
5401 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5402 Args.push_back(Builder.getFalse());
5403
5404 // Insert the declaration for the right overload types
5405 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5406 F->getParent(), F->getIntrinsicID(), OverloadTys);
5407
5408 // Copy operand bundles if any
5409 SmallVector<OperandBundleDef, 1> Bundles;
5410 CI->getOperandBundlesAsDefs(Bundles);
5411
5412 // Create the new call and copy calling properties
5413 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5414 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5415 NewCall->setCallingConv(CI->getCallingConv());
5416 NewCall->setAttributes(CI->getAttributes());
5417 NewCall->copyMetadata(*CI);
5418 return NewCall;
5419 };
5420
5421 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5422 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5423 "intrinsic should have 7 arguments");
5424 Type *T1 = CI->getArgOperand(4)->getType();
5425 Type *T2 = CI->getArgOperand(1)->getType();
5426 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5427 }
5428 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5429 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5430 "intrinsic should have 8 arguments");
5431 Type *T1 = CI->getArgOperand(4)->getType();
5432 Type *T2 = CI->getArgOperand(1)->getType();
5433 Type *T3 = CI->getArgOperand(3)->getType();
5434 Type *T4 = CI->getArgOperand(5)->getType();
5435 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5436 }
5437
5438 switch (F->getIntrinsicID()) {
5439 default:
5440 break;
5441 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5442 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5443 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5444 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5445 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5446 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5447 // Drop src0 and src1 modifiers.
5448 const Value *Op0 = CI->getArgOperand(0);
5449 const Value *Op2 = CI->getArgOperand(2);
5450 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5451 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5452 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5453 if (!ModA->isZero() || !ModB->isZero())
5454 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5455
5456 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5457 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5458 Args.push_back(CI->getArgOperand(I));
5459
5460 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5461 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5462 Overloads.push_back(Args[3]->getType());
5463 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5464 F->getParent(), F->getIntrinsicID(), Overloads);
5465
5466 SmallVector<OperandBundleDef, 1> Bundles;
5467 CI->getOperandBundlesAsDefs(Bundles);
5468
5469 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5470 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5471 NewCall->setCallingConv(CI->getCallingConv());
5472 NewCall->setAttributes(CI->getAttributes());
5473 NewCall->copyMetadata(*CI);
5474 NewCall->takeName(CI);
5475 return NewCall;
5476 }
5477 }
5478
5479 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5480 Value *LHS = CI->getArgOperand(0);
5481 Value *RHS = CI->getArgOperand(1);
5482 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5483 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5484 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5485 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5486 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5487 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5488 NewCall->setCallingConv(CI->getCallingConv());
5489 NewCall->copyMetadata(*CI);
5490 NewCall->takeName(CI);
5491 return NewCall;
5492 }
5493
5494 if (Name.starts_with("addrspacecast.nonnull")) {
5495 if (CI->getNumOperands() < 2) // Malformed bitcode.
5496 return nullptr;
5497 Value *ASC = Builder.CreateAddrSpaceCast(
5498 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5499 ASC->takeName(CI);
5500 return ASC;
5501 }
5502
5503 AtomicRMWInst::BinOp RMWOp =
5504 StringSwitch<AtomicRMWInst::BinOp>(Name)
5505 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5506 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5507 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5508 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5509 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5510 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5511 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5512 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5513 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5514 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5515 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5516 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5517 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5518
5519 unsigned NumOperands = CI->getNumOperands();
5520 if (NumOperands < 3) // Malformed bitcode.
5521 return nullptr;
5522
5523 Value *Ptr = CI->getArgOperand(0);
5524 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5525 if (!PtrTy) // Malformed.
5526 return nullptr;
5527
5528 Value *Val = CI->getArgOperand(1);
5529 if (Val->getType() != CI->getType()) // Malformed.
5530 return nullptr;
5531
5532 ConstantInt *OrderArg = nullptr;
5533 bool IsVolatile = false;
5534
5535 // These should have 5 arguments (plus the callee). A separate version of the
5536 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5537 if (NumOperands > 3)
5538 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5539
5540 // Ignore scope argument at 3
5541
5542 if (NumOperands > 5) {
5543 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5544 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5545 }
5546
5547 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5548 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5549 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5550 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5551 Order = AtomicOrdering::SequentiallyConsistent;
5552
5553 LLVMContext &Ctx = F->getContext();
5554
5555 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5556 Type *RetTy = CI->getType();
5557 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5558 if (VT->getElementType()->isIntegerTy(16)) {
5559 VectorType *AsBF16 =
5560 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5561 Val = Builder.CreateBitCast(Val, AsBF16);
5562 }
5563 }
5564
5565 // The scope argument never really worked correctly. Use agent as the most
5566 // conservative option which should still always produce the instruction.
5567 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5568 AtomicRMWInst *RMW =
5569 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5570
5571 unsigned AddrSpace = PtrTy->getAddressSpace();
5572 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5573 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5574 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5575 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5576 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5577 }
5578
5579 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5580 MDBuilder MDB(F->getContext());
5581 MDNode *RangeNotPrivate =
5582 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5583 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5584 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5585 }
5586
5587 if (IsVolatile)
5588 RMW->setVolatile(true);
5589
5590 return Builder.CreateBitCast(RMW, RetTy);
5591}
5592
5593/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5594/// plain MDNode, as it's the verifier's job to check these are the correct
5595/// types later.
5596static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5597 if (Op < CI->arg_size()) {
5598 if (MetadataAsValue *MAV =
5599 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5600 Metadata *MD = MAV->getMetadata();
5601 return dyn_cast_if_present<MDNode>(MD);
5602 }
5603 }
5604 return nullptr;
5605}
5606
5607/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5608static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5609 if (Op < CI->arg_size())
5610 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5611 return MAV->getMetadata();
5612 return nullptr;
5613}
5614
5615/// Convert debug intrinsic calls to non-instruction debug records.
5616/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5617/// \p CI - The debug intrinsic call.
5618static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5619 DbgRecord *DR = nullptr;
5620 if (Name == "label") {
5621 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5622 } else if (Name == "assign") {
5623 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5624 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5625 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5626 unwrapMAVMetadataOp(CI, 4),
5627 /*The address is a Value ref, it will be stored as a Metadata */
5628 unwrapMAVOp(CI, 5));
5629 } else if (Name == "declare") {
5630 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5631 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5632 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5633 } else if (Name == "addr") {
5634 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5635 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5636 // Don't try to add something to the expression if it's not an expression.
5637 // Instead, allow the verifier to fail later.
5638 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5639 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5640 }
5641 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5642 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5643 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5644 } else if (Name == "value") {
5645 // An old version of dbg.value had an extra offset argument.
5646 unsigned VarOp = 1;
5647 unsigned ExprOp = 2;
5648 if (CI->arg_size() == 4) {
5649 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5650 // Nonzero offset dbg.values get dropped without a replacement.
5651 if (!Offset || !Offset->isNullValue())
5652 return;
5653 VarOp = 2;
5654 ExprOp = 3;
5655 }
5656 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5657 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5658 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5659 nullptr);
5660 }
5661 DR->setDebugLoc(CI->getDebugLoc());
5662 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5663 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5664}
5665
5666static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5667 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5668 if (!Offset)
5669 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5670 int64_t OffsetVal = Offset->getSExtValue();
5671 return Builder.CreateIntrinsic(OffsetVal >= 0
5672 ? Intrinsic::vector_splice_left
5673 : Intrinsic::vector_splice_right,
5674 CI->getType(),
5675 {CI->getArgOperand(0), CI->getArgOperand(1),
5676 Builder.getInt32(std::abs(OffsetVal))});
5677}
5678
5679static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5680 Function *F, IRBuilder<> &Builder) {
5681 if (Name.starts_with("to.fp16")) {
5682 Value *Cast =
5683 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5684 return Builder.CreateBitCast(Cast, CI->getType());
5685 }
5686
5687 if (Name.starts_with("from.fp16")) {
5688 Value *Cast =
5689 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5690 return Builder.CreateFPExt(Cast, CI->getType());
5691 }
5692
5693 return nullptr;
5694}
5695
5696static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5697 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5698 if (!MD || !isa<MDString>(MD))
5699 return ICmpInst::BAD_ICMP_PREDICATE;
5700 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5701 .Case("eq", ICmpInst::ICMP_EQ)
5702 .Case("ne", ICmpInst::ICMP_NE)
5703 .Case("ugt", ICmpInst::ICMP_UGT)
5704 .Case("uge", ICmpInst::ICMP_UGE)
5705 .Case("ult", ICmpInst::ICMP_ULT)
5706 .Case("ule", ICmpInst::ICMP_ULE)
5707 .Case("sgt", ICmpInst::ICMP_SGT)
5708 .Case("sge", ICmpInst::ICMP_SGE)
5709 .Case("slt", ICmpInst::ICMP_SLT)
5710 .Case("sle", ICmpInst::ICMP_SLE)
5711 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5712}
5713
5714static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5715 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5716 if (!MD || !isa<MDString>(MD))
5717 return FCmpInst::BAD_FCMP_PREDICATE;
5718 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5719 .Case("oeq", FCmpInst::FCMP_OEQ)
5720 .Case("ogt", FCmpInst::FCMP_OGT)
5721 .Case("oge", FCmpInst::FCMP_OGE)
5722 .Case("olt", FCmpInst::FCMP_OLT)
5723 .Case("ole", FCmpInst::FCMP_OLE)
5724 .Case("one", FCmpInst::FCMP_ONE)
5725 .Case("ord", FCmpInst::FCMP_ORD)
5726 .Case("uno", FCmpInst::FCMP_UNO)
5727 .Case("ueq", FCmpInst::FCMP_UEQ)
5728 .Case("ugt", FCmpInst::FCMP_UGT)
5729 .Case("uge", FCmpInst::FCMP_UGE)
5730 .Case("ult", FCmpInst::FCMP_ULT)
5731 .Case("ule", FCmpInst::FCMP_ULE)
5732 .Case("une", FCmpInst::FCMP_UNE)
5733 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5734}
5735
5736static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5737 IRBuilder<> &Builder) {
5738 Value *Rep;
5739 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5740 if (Opcode && Instruction::isUnaryOp(Opcode))
5741 Rep =
5742 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5743 else if (Opcode && Instruction::isBinaryOp(Opcode))
5744 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5745 CI->getArgOperand(0), CI->getArgOperand(1));
5746 else if (Opcode && Instruction::isCast(Opcode))
5747 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5748 CI->getType());
5749 else if (Opcode == Instruction::ICmp)
5750 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5751 CI->getArgOperand(0), CI->getArgOperand(1));
5752 else if (Opcode == Instruction::FCmp)
5753 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5754 CI->getArgOperand(0), CI->getArgOperand(1));
5755 else if (Opcode == Instruction::Select)
5756 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5757 CI->getArgOperand(2));
5758 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5759 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5760 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5761 } else
5762 llvm_unreachable("Unexpected vp intrinsic");
5763 Rep->takeName(CI);
5764 return Rep;
5765}
5766
5767static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5768 IRBuilder<> &Builder) {
5769 Intrinsic::ID IID = NewFn->getIntrinsicID();
5770
5771 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5772 if (Defaults.empty())
5773 return false;
5774
5775 unsigned OldArgCount = CI->arg_size();
5776 unsigned NewArgCount = NewFn->arg_size();
5777
5778 if (OldArgCount < FirstDefault)
5779 return false;
5780
5781 // More arguments than the new intrinsic accepts, cannot upgrade.
5782 if (OldArgCount > NewArgCount)
5783 return false;
5784
5785 // The call already passes the defaulted arguments explicitly; only the
5786 // callee is still the old, shorter declaration, so retarget it.
5787 if (OldArgCount == NewArgCount) {
5788 if (CI->getFunctionType() != NewFn->getFunctionType())
5789 return false;
5790 CI->setCalledFunction(NewFn);
5791 return true;
5792 }
5793
5794 // OldArgCount < NewArgCount: Fill in each missing trailing default
5795 // argument from the table.
5796 SmallVector<Value *, 8> NewArgs(CI->args());
5797
5798 FunctionType *NewFT = NewFn->getFunctionType();
5799 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5800 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5801 "missing argument outside the default range");
5802 Type *ParamTy = NewFT->getParamType(Idx);
5803
5804 // Only integer types are supported (i1, i8, i16, i32, i64).
5805 if (!ParamTy->isIntegerTy())
5806 return false;
5807 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5808 }
5809
5810 // Preserve operand bundles by creating the call with them.
5811 SmallVector<OperandBundleDef, 1> OpBundles;
5812 CI->getOperandBundlesAsDefs(OpBundles);
5813 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5814
5815 NewCall->takeName(CI);
5816 NewCall->setCallingConv(CI->getCallingConv());
5817 NewCall->copyMetadata(*CI);
5818 if (auto *OldCI = dyn_cast<CallInst>(CI))
5819 NewCall->setTailCallKind(OldCI->getTailCallKind());
5820
5821 CI->replaceAllUsesWith(NewCall);
5822 CI->eraseFromParent();
5823 return true;
5824}
5825
5826/// Upgrade a call to an old intrinsic. All argument and return casting must be
5827/// provided to seamlessly integrate with existing context.
5828void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5829 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5830 // checks the callee's function type matches. It's likely we need to handle
5831 // type changes here.
5832 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5833 if (!F)
5834 return;
5835
5836 LLVMContext &C = CI->getContext();
5837 IRBuilder<> Builder(CI->getIterator());
5838 if (isa<FPMathOperator>(CI))
5839 Builder.setFastMathFlags(CI->getFastMathFlags());
5840
5841 if (!NewFn) {
5842 // Get the Function's name.
5843 StringRef Name = F->getName();
5844 if (!Name.consume_front("llvm."))
5845 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5846
5847 bool IsX86 = Name.consume_front("x86.");
5848 bool IsNVVM = Name.consume_front("nvvm.");
5849 bool IsAArch64 = Name.consume_front("aarch64.");
5850 bool IsARM = Name.consume_front("arm.");
5851 bool IsAMDGCN = Name.consume_front("amdgcn.");
5852 bool IsDbg = Name.consume_front("dbg.");
5853 bool IsOldSplice =
5854 (Name.consume_front("experimental.vector.splice") ||
5855 Name.consume_front("vector.splice")) &&
5856 !(Name.starts_with(".left") || Name.starts_with(".right"));
5857 Value *Rep = nullptr;
5858
5859 if (!IsX86 && Name == "stackprotectorcheck") {
5860 Rep = nullptr;
5861 } else if (IsNVVM) {
5862 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5863 } else if (IsX86) {
5864 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5865 } else if (IsAArch64) {
5866 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5867 } else if (IsARM) {
5868 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5869 } else if (IsAMDGCN) {
5870 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5871 } else if (IsDbg) {
5872 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5873 } else if (IsOldSplice) {
5874 Rep = upgradeVectorSplice(CI, Builder);
5875 } else if (Name.consume_front("convert.")) {
5876 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5877 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5878 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5879 Rep = nullptr;
5880 } else if (shouldUpgradeVPIntrinsic(Name)) {
5881 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5882 } else {
5883 llvm_unreachable("Unknown function for CallBase upgrade.");
5884 }
5885
5886 if (Rep)
5887 CI->replaceAllUsesWith(Rep);
5888 CI->eraseFromParent();
5889 return;
5890 }
5891
5892 const auto &DefaultCase = [&]() -> void {
5893 if (F == NewFn)
5894 return;
5895
5896 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5897 // Handle generic mangling change.
5898 assert(
5899 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5900 "Unknown function for CallBase upgrade and isn't just a name change");
5901 CI->setCalledFunction(NewFn);
5902 return;
5903 }
5904
5905 // This must be an upgrade from a named to a literal struct.
5906 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5907 assert(OldST != NewFn->getReturnType() &&
5908 "Return type must have changed");
5909 assert(OldST->getNumElements() ==
5910 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5911 "Must have same number of elements");
5912
5913 SmallVector<Value *> Args(CI->args());
5914 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5915 NewCI->setAttributes(CI->getAttributes());
5916 Value *Res = PoisonValue::get(OldST);
5917 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5918 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5919 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5920 }
5921 CI->replaceAllUsesWith(Res);
5922 CI->eraseFromParent();
5923 return;
5924 }
5925
5926 // We're probably about to produce something invalid. Let the verifier catch
5927 // it instead of dying here.
5928 CI->setCalledOperand(
5929 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5930 return;
5931 };
5932 CallInst *NewCall = nullptr;
5933 switch (NewFn->getIntrinsicID()) {
5934 default: {
5935 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5936 return;
5937 DefaultCase();
5938 return;
5939 }
5940 case Intrinsic::arm_neon_vst1:
5941 case Intrinsic::arm_neon_vst2:
5942 case Intrinsic::arm_neon_vst3:
5943 case Intrinsic::arm_neon_vst4:
5944 case Intrinsic::arm_neon_vst2lane:
5945 case Intrinsic::arm_neon_vst3lane:
5946 case Intrinsic::arm_neon_vst4lane: {
5947 SmallVector<Value *, 4> Args(CI->args());
5948 NewCall = Builder.CreateCall(NewFn, Args);
5949 break;
5950 }
5951 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5952 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5953 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5954 LLVMContext &Ctx = F->getParent()->getContext();
5955 SmallVector<Value *, 4> Args(CI->args());
5956 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5957 cast<ConstantInt>(Args[3])->getZExtValue());
5958 NewCall = Builder.CreateCall(NewFn, Args);
5959 break;
5960 }
5961 case Intrinsic::aarch64_sve_ld3_sret:
5962 case Intrinsic::aarch64_sve_ld4_sret:
5963 case Intrinsic::aarch64_sve_ld2_sret: {
5964 // Is this a trivial remangle of the name to support ptr address spaces?
5965 if (isa<StructType>(F->getReturnType())) {
5966 DefaultCase();
5967 return;
5968 }
5969
5970 StringRef Name = F->getName();
5971 Name = Name.substr(5);
5972 unsigned N = StringSwitch<unsigned>(Name)
5973 .StartsWith("aarch64.sve.ld2", 2)
5974 .StartsWith("aarch64.sve.ld3", 3)
5975 .StartsWith("aarch64.sve.ld4", 4)
5976 .Default(0);
5977 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5978 unsigned MinElts = RetTy->getMinNumElements() / N;
5979 SmallVector<Value *, 2> Args(CI->args());
5980 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5981 Value *Ret = llvm::PoisonValue::get(RetTy);
5982 for (unsigned I = 0; I < N; I++) {
5983 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5984 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5985 }
5986 NewCall = dyn_cast<CallInst>(Ret);
5987 break;
5988 }
5989
5990 case Intrinsic::coro_end_async:
5991 case Intrinsic::coro_end: {
5992 SmallVector<Value *, 3> Args(CI->args());
5993 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5994 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5995 NewCall = Builder.CreateCall(NewFn, Args);
5996
5997 if (!CI->getType()->isVoidTy()) {
5998 if (!CI->use_empty()) {
5999 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
6000 CI->getModule(), Intrinsic::coro_is_in_ramp);
6001 Value *InRamp = Builder.CreateCall(IsInRamp);
6002 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
6003 }
6004 CI->eraseFromParent();
6005 return;
6006 }
6007
6008 break;
6009 }
6010
6011 case Intrinsic::vector_extract: {
6012 StringRef Name = F->getName();
6013 Name = Name.substr(5); // Strip llvm
6014 if (!Name.starts_with("aarch64.sve.tuple.get")) {
6015 DefaultCase();
6016 return;
6017 }
6018 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6019 unsigned MinElts = RetTy->getMinNumElements();
6020 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6021 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
6022 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
6023 break;
6024 }
6025
6026 case Intrinsic::vector_insert: {
6027 StringRef Name = F->getName();
6028 Name = Name.substr(5);
6029 if (!Name.starts_with("aarch64.sve.tuple")) {
6030 DefaultCase();
6031 return;
6032 }
6033 if (Name.starts_with("aarch64.sve.tuple.set")) {
6034 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6035 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
6036 Value *NewIdx =
6037 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
6038 NewCall = Builder.CreateCall(
6039 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
6040 break;
6041 }
6042 if (Name.starts_with("aarch64.sve.tuple.create")) {
6043 unsigned N = StringSwitch<unsigned>(Name)
6044 .StartsWith("aarch64.sve.tuple.create2", 2)
6045 .StartsWith("aarch64.sve.tuple.create3", 3)
6046 .StartsWith("aarch64.sve.tuple.create4", 4)
6047 .Default(0);
6048 assert(N > 1 && "Create is expected to be between 2-4");
6049 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6050 Value *Ret = llvm::PoisonValue::get(RetTy);
6051 unsigned MinElts = RetTy->getMinNumElements() / N;
6052 for (unsigned I = 0; I < N; I++) {
6053 Value *V = CI->getArgOperand(I);
6054 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
6055 }
6056 NewCall = dyn_cast<CallInst>(Ret);
6057 }
6058 break;
6059 }
6060
6061 case Intrinsic::arm_neon_bfdot:
6062 case Intrinsic::arm_neon_bfmmla:
6063 case Intrinsic::arm_neon_bfmlalb:
6064 case Intrinsic::arm_neon_bfmlalt:
6065 case Intrinsic::aarch64_neon_bfdot:
6066 case Intrinsic::aarch64_neon_bfmmla:
6067 case Intrinsic::aarch64_neon_bfmlalb:
6068 case Intrinsic::aarch64_neon_bfmlalt: {
6069 SmallVector<Value *, 3> Args;
6070 assert(CI->arg_size() == 3 &&
6071 "Mismatch between function args and call args");
6072 size_t OperandWidth =
6073 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
6074 assert((OperandWidth == 64 || OperandWidth == 128) &&
6075 "Unexpected operand width");
6076 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6077 auto Iter = CI->args().begin();
6078 Args.push_back(*Iter++);
6079 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6080 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6081 NewCall = Builder.CreateCall(NewFn, Args);
6082 break;
6083 }
6084
6085 case Intrinsic::bitreverse:
6086 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6087 break;
6088
6089 case Intrinsic::ctlz:
6090 case Intrinsic::cttz: {
6091 if (CI->arg_size() != 1) {
6092 DefaultCase();
6093 return;
6094 }
6095
6096 NewCall =
6097 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6098 break;
6099 }
6100
6101 case Intrinsic::objectsize: {
6102 Value *NullIsUnknownSize =
6103 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6104 Value *Dynamic =
6105 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6106 NewCall = Builder.CreateCall(
6107 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6108 break;
6109 }
6110
6111 case Intrinsic::ctpop:
6112 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6113 break;
6114 case Intrinsic::dbg_value: {
6115 StringRef Name = F->getName();
6116 Name = Name.substr(5); // Strip llvm.
6117 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6118 if (Name.starts_with("dbg.addr")) {
6119 DIExpression *Expr = cast<DIExpression>(
6120 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6121 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6122 NewCall =
6123 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6124 MetadataAsValue::get(C, Expr)});
6125 break;
6126 }
6127
6128 // Upgrade from the old version that had an extra offset argument.
6129 assert(CI->arg_size() == 4);
6130 // Drop nonzero offsets instead of attempting to upgrade them.
6131 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6132 if (Offset->isNullValue()) {
6133 NewCall = Builder.CreateCall(
6134 NewFn,
6135 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6136 break;
6137 }
6138 CI->eraseFromParent();
6139 return;
6140 }
6141
6142 case Intrinsic::ptr_annotation:
6143 // Upgrade from versions that lacked the annotation attribute argument.
6144 if (CI->arg_size() != 4) {
6145 DefaultCase();
6146 return;
6147 }
6148
6149 // Create a new call with an added null annotation attribute argument.
6150 NewCall = Builder.CreateCall(
6151 NewFn,
6152 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6153 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6154 NewCall->takeName(CI);
6155 CI->replaceAllUsesWith(NewCall);
6156 CI->eraseFromParent();
6157 return;
6158
6159 case Intrinsic::var_annotation:
6160 // Upgrade from versions that lacked the annotation attribute argument.
6161 if (CI->arg_size() != 4) {
6162 DefaultCase();
6163 return;
6164 }
6165 // Create a new call with an added null annotation attribute argument.
6166 NewCall = Builder.CreateCall(
6167 NewFn,
6168 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6169 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6170 NewCall->takeName(CI);
6171 CI->replaceAllUsesWith(NewCall);
6172 CI->eraseFromParent();
6173 return;
6174
6175 case Intrinsic::riscv_aes32dsi:
6176 case Intrinsic::riscv_aes32dsmi:
6177 case Intrinsic::riscv_aes32esi:
6178 case Intrinsic::riscv_aes32esmi:
6179 case Intrinsic::riscv_sm4ks:
6180 case Intrinsic::riscv_sm4ed: {
6181 // The last argument to these intrinsics used to be i8 and changed to i32.
6182 // The type overload for sm4ks and sm4ed was removed.
6183 Value *Arg2 = CI->getArgOperand(2);
6184 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6185 return;
6186
6187 Value *Arg0 = CI->getArgOperand(0);
6188 Value *Arg1 = CI->getArgOperand(1);
6189 if (CI->getType()->isIntegerTy(64)) {
6190 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6191 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6192 }
6193
6194 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6195 cast<ConstantInt>(Arg2)->getZExtValue());
6196
6197 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6198 Value *Res = NewCall;
6199 if (Res->getType() != CI->getType())
6200 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6201 NewCall->takeName(CI);
6202 CI->replaceAllUsesWith(Res);
6203 CI->eraseFromParent();
6204 return;
6205 }
6206 case Intrinsic::nvvm_mapa_shared_cluster: {
6207 // Create a new call with the correct address space.
6208 NewCall =
6209 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6210 Value *Res = NewCall;
6211 Res = Builder.CreateAddrSpaceCast(
6212 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6213 NewCall->takeName(CI);
6214 CI->replaceAllUsesWith(Res);
6215 CI->eraseFromParent();
6216 return;
6217 }
6218 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6219 SmallVector<Value *, 4> Args(CI->args());
6220 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6221 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6222 Args[0] = Builder.CreateAddrSpaceCast(
6223 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6224
6225 // Append the missing trailing flag_valid_pattern (0 = disabled).
6226 Args.push_back(Builder.getInt32(0));
6227
6228 NewCall = Builder.CreateCall(NewFn, Args);
6229 NewCall->takeName(CI);
6230 CI->replaceAllUsesWith(NewCall);
6231 CI->eraseFromParent();
6232 return;
6233 }
6234 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6235 // (dst, mbar, src, size, ch, flag_ch)
6236 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6237 // i32 0 /* flag_valid_pattern=disabled */)
6238 SmallVector<Value *, 10> Args;
6239 for (unsigned I = 0; I < 4; ++I)
6240 Args.push_back(CI->getArgOperand(I));
6241 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6242 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6243 Args.push_back(CI->getArgOperand(4)); // cache_hint
6244 Args.push_back(CI->getArgOperand(5)); // flag_ch
6245 Args.push_back(Builder.getInt1(false)); // flag_oob
6246 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6247
6248 NewCall = Builder.CreateCall(NewFn, Args);
6249 NewCall->takeName(CI);
6250 CI->replaceAllUsesWith(NewCall);
6251 CI->eraseFromParent();
6252 return;
6253 }
6254 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6255 // Create a new call with the correct address space.
6256 SmallVector<Value *, 4> Args(CI->args());
6257 Args[0] = Builder.CreateAddrSpaceCast(
6258 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6259
6260 NewCall = Builder.CreateCall(NewFn, Args);
6261 NewCall->takeName(CI);
6262 CI->replaceAllUsesWith(NewCall);
6263 CI->eraseFromParent();
6264 return;
6265 }
6266 // clang-format off
6267#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6268 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6269 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6270#undef G2S_CLUSTER_CASE
6271 {
6272 SmallVector<Value *, 16> Args(CI->args());
6273 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6274 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6275 Args[0] = Builder.CreateAddrSpaceCast(
6276 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6277
6278 // Append the missing trailing arguments with default values (cta_group,
6279 // flag_valid_pattern).
6280 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6281 Args.push_back(Builder.getInt32(0));
6282
6283 NewCall = Builder.CreateCall(NewFn, Args);
6284 NewCall->takeName(CI);
6285 CI->replaceAllUsesWith(NewCall);
6286 CI->eraseFromParent();
6287 return;
6288 }
6289
6290#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6291 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6292 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6293#undef G2S_CTA_CASE
6294 {
6295 SmallVector<Value *, 16> Args(CI->args());
6296 // Append the missing trailing flag_valid_pattern argument with default
6297 // value 0.
6298 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6299 "expected only the trailing flag_valid_pattern to be missing");
6300 Args.push_back(Builder.getInt32(0));
6301
6302 NewCall = Builder.CreateCall(NewFn, Args);
6303 NewCall->takeName(CI);
6304 CI->replaceAllUsesWith(NewCall);
6305 CI->eraseFromParent();
6306 return;
6307 }
6308#undef NVVM_TMA_G2S_MODES
6309 // clang-format on
6310
6311 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6312 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6313 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6314 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6315 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6316 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6317 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6318 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6319 StringRef Name = F->getName();
6320 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6321 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6322
6323 SmallVector<Value *, 16> Args(CI->args());
6324 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6325 NewCall = Builder.CreateCall(NewFn, Args);
6326 break;
6327 }
6328 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6329 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6330 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6331 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6332 NewCall =
6333 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6334 Builder.getFalse()});
6335 break;
6336 case Intrinsic::nvvm_mbarrier_init: {
6337 SmallVector<Value *, 3> Args(CI->args());
6338 // The .shared variant folded into the overloaded form without gaining an
6339 // operand, so only the pre-layout two-argument form needs one appended.
6340 if (Args.size() == 2)
6341 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6342 NewCall = Builder.CreateCall(NewFn, Args);
6343 break;
6344 }
6345 case Intrinsic::riscv_sha256sig0:
6346 case Intrinsic::riscv_sha256sig1:
6347 case Intrinsic::riscv_sha256sum0:
6348 case Intrinsic::riscv_sha256sum1:
6349 case Intrinsic::riscv_sm3p0:
6350 case Intrinsic::riscv_sm3p1: {
6351 // The last argument to these intrinsics used to be i8 and changed to i32.
6352 // The type overload for sm4ks and sm4ed was removed.
6353 if (!CI->getType()->isIntegerTy(64))
6354 return;
6355
6356 Value *Arg =
6357 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6358
6359 NewCall = Builder.CreateCall(NewFn, Arg);
6360 Value *Res =
6361 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6362 NewCall->takeName(CI);
6363 CI->replaceAllUsesWith(Res);
6364 CI->eraseFromParent();
6365 return;
6366 }
6367
6368 case Intrinsic::x86_xop_vfrcz_ss:
6369 case Intrinsic::x86_xop_vfrcz_sd:
6370 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6371 break;
6372
6373 case Intrinsic::x86_xop_vpermil2pd:
6374 case Intrinsic::x86_xop_vpermil2ps:
6375 case Intrinsic::x86_xop_vpermil2pd_256:
6376 case Intrinsic::x86_xop_vpermil2ps_256: {
6377 SmallVector<Value *, 4> Args(CI->args());
6378 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6379 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6380 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6381 NewCall = Builder.CreateCall(NewFn, Args);
6382 break;
6383 }
6384
6385 case Intrinsic::x86_sse41_ptestc:
6386 case Intrinsic::x86_sse41_ptestz:
6387 case Intrinsic::x86_sse41_ptestnzc: {
6388 // The arguments for these intrinsics used to be v4f32, and changed
6389 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6390 // So, the only thing required is a bitcast for both arguments.
6391 // First, check the arguments have the old type.
6392 Value *Arg0 = CI->getArgOperand(0);
6393 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6394 return;
6395
6396 // Old intrinsic, add bitcasts
6397 Value *Arg1 = CI->getArgOperand(1);
6398
6399 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6400
6401 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6402 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6403
6404 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6405 break;
6406 }
6407
6408 case Intrinsic::x86_rdtscp: {
6409 // This used to take 1 arguments. If we have no arguments, it is already
6410 // upgraded.
6411 if (CI->getNumOperands() == 0)
6412 return;
6413
6414 NewCall = Builder.CreateCall(NewFn);
6415 // Extract the second result and store it.
6416 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6417 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6418 // Replace the original call result with the first result of the new call.
6419 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6420
6421 NewCall->takeName(CI);
6422 CI->replaceAllUsesWith(TSC);
6423 CI->eraseFromParent();
6424 return;
6425 }
6426
6427 case Intrinsic::x86_sse41_insertps:
6428 case Intrinsic::x86_sse41_dppd:
6429 case Intrinsic::x86_sse41_dpps:
6430 case Intrinsic::x86_sse41_mpsadbw:
6431 case Intrinsic::x86_avx_dp_ps_256:
6432 case Intrinsic::x86_avx2_mpsadbw: {
6433 // Need to truncate the last argument from i32 to i8 -- this argument models
6434 // an inherently 8-bit immediate operand to these x86 instructions.
6435 SmallVector<Value *, 4> Args(CI->args());
6436
6437 // Replace the last argument with a trunc.
6438 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6439 NewCall = Builder.CreateCall(NewFn, Args);
6440 break;
6441 }
6442
6443 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6444 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6445 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6446 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6447 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6448 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6449 SmallVector<Value *, 4> Args(CI->args());
6450 unsigned NumElts =
6451 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6452 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6453
6454 NewCall = Builder.CreateCall(NewFn, Args);
6455 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6456
6457 NewCall->takeName(CI);
6458 CI->replaceAllUsesWith(Res);
6459 CI->eraseFromParent();
6460 return;
6461 }
6462
6463 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6464 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6465 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6466 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6467 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6468 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6469 SmallVector<Value *, 4> Args(CI->args());
6470 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6471 if (NewFn->getIntrinsicID() ==
6472 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6473 Args[1] = Builder.CreateBitCast(
6474 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6475
6476 NewCall = Builder.CreateCall(NewFn, Args);
6477 Value *Res = Builder.CreateBitCast(
6478 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6479
6480 NewCall->takeName(CI);
6481 CI->replaceAllUsesWith(Res);
6482 CI->eraseFromParent();
6483 return;
6484 }
6485 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6486 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6487 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6488 SmallVector<Value *, 4> Args(CI->args());
6489 unsigned NumElts =
6490 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6491 Args[1] = Builder.CreateBitCast(
6492 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6493 Args[2] = Builder.CreateBitCast(
6494 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6495
6496 NewCall = Builder.CreateCall(NewFn, Args);
6497 break;
6498 }
6499
6500 case Intrinsic::thread_pointer: {
6501 NewCall = Builder.CreateCall(NewFn, {});
6502 break;
6503 }
6504
6505 case Intrinsic::memcpy:
6506 case Intrinsic::memmove:
6507 case Intrinsic::memset: {
6508 // We have to make sure that the call signature is what we're expecting.
6509 // We only want to change the old signatures by removing the alignment arg:
6510 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6511 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6512 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6513 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6514 // Note: i8*'s in the above can be any pointer type
6515 if (CI->arg_size() != 5) {
6516 DefaultCase();
6517 return;
6518 }
6519 // Remove alignment argument (3), and add alignment attributes to the
6520 // dest/src pointers.
6521 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6522 CI->getArgOperand(2), CI->getArgOperand(4)};
6523 NewCall = Builder.CreateCall(NewFn, Args);
6524 AttributeList OldAttrs = CI->getAttributes();
6525 AttributeList NewAttrs = AttributeList::get(
6526 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6527 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6528 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6529 NewCall->setAttributes(NewAttrs);
6530 auto *MemCI = cast<MemIntrinsic>(NewCall);
6531 // All mem intrinsics support dest alignment.
6532 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6533 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6534 // Memcpy/Memmove also support source alignment.
6535 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6536 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6537 break;
6538 }
6539
6540 case Intrinsic::masked_load:
6541 case Intrinsic::masked_gather:
6542 case Intrinsic::masked_store:
6543 case Intrinsic::masked_scatter: {
6544 if (CI->arg_size() != 4) {
6545 DefaultCase();
6546 return;
6547 }
6548
6549 auto GetMaybeAlign = [](Value *Op) {
6550 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6551 uint64_t Val = CI->getZExtValue();
6552 if (Val == 0)
6553 return MaybeAlign();
6554 if (isPowerOf2_64(Val))
6555 return MaybeAlign(Val);
6556 }
6557 reportFatalUsageError("Invalid alignment argument");
6558 };
6559 auto GetAlign = [&](Value *Op) {
6560 MaybeAlign Align = GetMaybeAlign(Op);
6561 if (Align)
6562 return *Align;
6563 reportFatalUsageError("Invalid zero alignment argument");
6564 };
6565
6566 const DataLayout &DL = CI->getDataLayout();
6567 switch (NewFn->getIntrinsicID()) {
6568 case Intrinsic::masked_load:
6569 NewCall = Builder.CreateMaskedLoad(
6570 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6571 CI->getArgOperand(2), CI->getArgOperand(3));
6572 break;
6573 case Intrinsic::masked_gather:
6574 NewCall = Builder.CreateMaskedGather(
6575 CI->getType(), CI->getArgOperand(0),
6576 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6577 CI->getType()->getScalarType()),
6578 CI->getArgOperand(2), CI->getArgOperand(3));
6579 break;
6580 case Intrinsic::masked_store:
6581 NewCall = Builder.CreateMaskedStore(
6582 CI->getArgOperand(0), CI->getArgOperand(1),
6583 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6584 break;
6585 case Intrinsic::masked_scatter:
6586 NewCall = Builder.CreateMaskedScatter(
6587 CI->getArgOperand(0), CI->getArgOperand(1),
6588 DL.getValueOrABITypeAlignment(
6589 GetMaybeAlign(CI->getArgOperand(2)),
6590 CI->getArgOperand(0)->getType()->getScalarType()),
6591 CI->getArgOperand(3));
6592 break;
6593 default:
6594 llvm_unreachable("Unexpected intrinsic ID");
6595 }
6596 // Previous metadata is still valid.
6597 NewCall->copyMetadata(*CI);
6598 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6599 break;
6600 }
6601
6602 case Intrinsic::lifetime_start:
6603 case Intrinsic::lifetime_end: {
6604 if (CI->arg_size() != 2) {
6605 DefaultCase();
6606 return;
6607 }
6608
6609 Value *Ptr = CI->getArgOperand(1);
6610 // Try to strip pointer casts, such that the lifetime works on an alloca.
6611 Ptr = Ptr->stripPointerCasts();
6612 if (isa<AllocaInst>(Ptr)) {
6613 // Don't use NewFn, as we might have looked through an addrspacecast.
6614 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6615 NewCall = Builder.CreateLifetimeStart(Ptr);
6616 else
6617 NewCall = Builder.CreateLifetimeEnd(Ptr);
6618 break;
6619 }
6620
6621 // Otherwise remove the lifetime marker.
6622 CI->eraseFromParent();
6623 return;
6624 }
6625
6626 case Intrinsic::x86_avx512_vpdpbusd_128:
6627 case Intrinsic::x86_avx512_vpdpbusd_256:
6628 case Intrinsic::x86_avx512_vpdpbusd_512:
6629 case Intrinsic::x86_avx512_vpdpbusds_128:
6630 case Intrinsic::x86_avx512_vpdpbusds_256:
6631 case Intrinsic::x86_avx512_vpdpbusds_512:
6632 case Intrinsic::x86_avx2_vpdpbssd_128:
6633 case Intrinsic::x86_avx2_vpdpbssd_256:
6634 case Intrinsic::x86_avx10_vpdpbssd_512:
6635 case Intrinsic::x86_avx2_vpdpbssds_128:
6636 case Intrinsic::x86_avx2_vpdpbssds_256:
6637 case Intrinsic::x86_avx10_vpdpbssds_512:
6638 case Intrinsic::x86_avx2_vpdpbsud_128:
6639 case Intrinsic::x86_avx2_vpdpbsud_256:
6640 case Intrinsic::x86_avx10_vpdpbsud_512:
6641 case Intrinsic::x86_avx2_vpdpbsuds_128:
6642 case Intrinsic::x86_avx2_vpdpbsuds_256:
6643 case Intrinsic::x86_avx10_vpdpbsuds_512:
6644 case Intrinsic::x86_avx2_vpdpbuud_128:
6645 case Intrinsic::x86_avx2_vpdpbuud_256:
6646 case Intrinsic::x86_avx10_vpdpbuud_512:
6647 case Intrinsic::x86_avx2_vpdpbuuds_128:
6648 case Intrinsic::x86_avx2_vpdpbuuds_256:
6649 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6650 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6651 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6652 CI->getArgOperand(2)};
6653 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6654 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6655 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6656
6657 NewCall = Builder.CreateCall(NewFn, Args);
6658 break;
6659 }
6660 case Intrinsic::x86_avx512_vpdpwssd_128:
6661 case Intrinsic::x86_avx512_vpdpwssd_256:
6662 case Intrinsic::x86_avx512_vpdpwssd_512:
6663 case Intrinsic::x86_avx512_vpdpwssds_128:
6664 case Intrinsic::x86_avx512_vpdpwssds_256:
6665 case Intrinsic::x86_avx512_vpdpwssds_512:
6666 case Intrinsic::x86_avx2_vpdpwsud_128:
6667 case Intrinsic::x86_avx2_vpdpwsud_256:
6668 case Intrinsic::x86_avx10_vpdpwsud_512:
6669 case Intrinsic::x86_avx2_vpdpwsuds_128:
6670 case Intrinsic::x86_avx2_vpdpwsuds_256:
6671 case Intrinsic::x86_avx10_vpdpwsuds_512:
6672 case Intrinsic::x86_avx2_vpdpwusd_128:
6673 case Intrinsic::x86_avx2_vpdpwusd_256:
6674 case Intrinsic::x86_avx10_vpdpwusd_512:
6675 case Intrinsic::x86_avx2_vpdpwusds_128:
6676 case Intrinsic::x86_avx2_vpdpwusds_256:
6677 case Intrinsic::x86_avx10_vpdpwusds_512:
6678 case Intrinsic::x86_avx2_vpdpwuud_128:
6679 case Intrinsic::x86_avx2_vpdpwuud_256:
6680 case Intrinsic::x86_avx10_vpdpwuud_512:
6681 case Intrinsic::x86_avx2_vpdpwuuds_128:
6682 case Intrinsic::x86_avx2_vpdpwuuds_256:
6683 case Intrinsic::x86_avx10_vpdpwuuds_512:
6684 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6685 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6686 CI->getArgOperand(2)};
6687 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6688 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6689 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6690
6691 NewCall = Builder.CreateCall(NewFn, Args);
6692 break;
6693 }
6694 assert(NewCall && "Should have either set this variable or returned through "
6695 "the default case");
6696 NewCall->takeName(CI);
6697 CI->replaceAllUsesWith(NewCall);
6698 CI->eraseFromParent();
6699}
6700
6701void llvm::UpgradeCallsToIntrinsic(Function *F) {
6702 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6703
6704 // Check if this function should be upgraded and get the replacement function
6705 // if there is one.
6706 Function *NewFn;
6707 if (UpgradeIntrinsicFunction(F, NewFn)) {
6708 // Replace all users of the old function with the new function or new
6709 // instructions. This is not a range loop because the call is deleted.
6710 for (User *U : make_early_inc_range(F->users()))
6711 if (CallBase *CB = dyn_cast<CallBase>(U))
6712 UpgradeIntrinsicCall(CB, NewFn);
6713
6714 // Remove old function, no longer used, from the module.
6715 if (F != NewFn)
6716 F->eraseFromParent();
6717 }
6718}
6719
6720MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6721 const unsigned NumOperands = MD.getNumOperands();
6722 if (NumOperands == 0)
6723 return &MD; // Invalid, punt to a verifier error.
6724
6725 // Check if the tag uses struct-path aware TBAA format.
6726 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6727 return &MD;
6728
6729 auto &Context = MD.getContext();
6730 if (NumOperands == 3) {
6731 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6732 MDNode *ScalarType = MDNode::get(Context, Elts);
6733 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6734 Metadata *Elts2[] = {ScalarType, ScalarType,
6735 ConstantAsMetadata::get(
6736 Constant::getNullValue(Type::getInt64Ty(Context))),
6737 MD.getOperand(2)};
6738 return MDNode::get(Context, Elts2);
6739 }
6740 // Create a MDNode <MD, MD, offset 0>
6741 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6742 Type::getInt64Ty(Context)))};
6743 return MDNode::get(Context, Elts);
6744}
6745
6746MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6747 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6748 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6749 unsigned NumOperands = MD.getNumOperands();
6750 if (NumOperands == 0 || NumOperands % 3 != 0)
6751 return &MD; // Malformed; leave it for the verifier to reject.
6752
6753 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6754 bool Changed = false;
6755 for (unsigned I = 2; I < NumOperands; I += 3) {
6756 auto *Tag = dyn_cast_or_null<MDNode>(Elts[I]);
6757 if (!Tag)
6758 continue;
6759 MDNode *Upgraded = UpgradeTBAANode(*Tag);
6760 if (Upgraded == Tag)
6761 continue;
6762 Elts[I] = Upgraded;
6763 Changed = true;
6764 }
6765 return Changed ? MDNode::get(MD.getContext(), Elts) : &MD;
6766}
6767
6768Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6769 Instruction *&Temp) {
6770 if (Opc != Instruction::BitCast)
6771 return nullptr;
6772
6773 Temp = nullptr;
6774 Type *SrcTy = V->getType();
6775 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6776 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6777 LLVMContext &Context = V->getContext();
6778
6779 // We have no information about target data layout, so we assume that
6780 // the maximum pointer size is 64bit.
6781 Type *MidTy = Type::getInt64Ty(Context);
6782 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6783
6784 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6785 }
6786
6787 return nullptr;
6788}
6789
6790Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6791 if (Opc != Instruction::BitCast)
6792 return nullptr;
6793
6794 Type *SrcTy = C->getType();
6795 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6796 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6797 LLVMContext &Context = C->getContext();
6798
6799 // We have no information about target data layout, so we assume that
6800 // the maximum pointer size is 64bit.
6801 Type *MidTy = Type::getInt64Ty(Context);
6802
6803 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6804 DestTy);
6805 }
6806
6807 return nullptr;
6808}
6809
6810static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6811 if (Flag.getNumOperands() < 3)
6812 return std::nullopt;
6813 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6814 return Name->getString();
6815 return std::nullopt;
6816}
6817
6818/// Check the debug info version number, if it is out-dated, drop the debug
6819/// info. Return true if module is modified.
6820bool llvm::UpgradeDebugInfo(Module &M) {
6821 if (!getIROptions(M.getContext()).auto_upgrade_debug_info)
6822 return false;
6823
6824 llvm::TimeTraceScope timeScope("Upgrade debug info");
6825 // We need to get metadata before the module is verified (i.e., getModuleFlag
6826 // makes assumptions that we haven't verified yet). Carefully extract the flag
6827 // from the metadata.
6828 unsigned Version = 0;
6829 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6830 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6831 if (auto Name = getModuleFlagNameSafely(*Flag))
6832 return *Name == "Debug Info Version";
6833 return false;
6834 });
6835 if (OpIt != ModFlags->op_end()) {
6836 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6837 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6838 Version = CI->getZExtValue();
6839 }
6840 }
6841
6842 if (Version == DEBUG_METADATA_VERSION) {
6843 bool BrokenDebugInfo = false;
6844 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6845 report_fatal_error("Broken module found, compilation aborted!");
6846 if (!BrokenDebugInfo)
6847 // Everything is ok.
6848 return false;
6849 else {
6850 // Diagnose malformed debug info.
6851 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6852 M.getContext().diagnose(Diag);
6853 }
6854 }
6855 bool Modified = StripDebugInfo(M);
6856 if (Modified && Version != DEBUG_METADATA_VERSION) {
6857 // Diagnose a version mismatch.
6858 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6859 M.getContext().diagnose(DiagVersion);
6860 }
6861 return Modified;
6862}
6863
6864static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6865 GlobalValue *GV, const Metadata *V) {
6866 Function *F = cast<Function>(GV);
6867
6868 constexpr StringLiteral DefaultValue = "1";
6869 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6870 unsigned Length = 0;
6871
6872 if (F->hasFnAttribute(Attr)) {
6873 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6874 // parse these elements placing them into Vect3
6875 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6876 for (; Length < 3 && !S.empty(); Length++) {
6877 auto [Part, Rest] = S.split(',');
6878 Vect3[Length] = Part.trim();
6879 S = Rest;
6880 }
6881 }
6882
6883 const unsigned Dim = DimC - 'x';
6884 assert(Dim < 3 && "Unexpected dim char");
6885
6886 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6887
6888 // local variable required for StringRef in Vect3 to point to.
6889 const std::string VStr = llvm::utostr(VInt);
6890 Vect3[Dim] = VStr;
6891 Length = std::max(Length, Dim + 1);
6892
6893 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6894 F->addFnAttr(Attr, NewAttr);
6895}
6896
6897static inline bool isXYZ(StringRef S) {
6898 return S == "x" || S == "y" || S == "z";
6899}
6900
6901bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6902 const Metadata *V) {
6903 if (K == "kernel") {
6904 if (!mdconst::extract<ConstantInt>(V)->isZero())
6905 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6906 return true;
6907 }
6908 if (K == "align") {
6909 // V is a bitfeild specifying two 16-bit values. The alignment value is
6910 // specfied in low 16-bits, The index is specified in the high bits. For the
6911 // index, 0 indicates the return value while higher values correspond to
6912 // each parameter (idx = param + 1).
6913 const uint64_t AlignIdxValuePair =
6914 mdconst::extract<ConstantInt>(V)->getZExtValue();
6915 const unsigned Idx = (AlignIdxValuePair >> 16);
6916 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6917 cast<Function>(GV)->addAttributeAtIndex(
6918 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6919 return true;
6920 }
6921 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6922 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6923 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6924 return true;
6925 }
6926 if (K == "minctasm") {
6927 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6928 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6929 return true;
6930 }
6931 if (K == "maxnreg") {
6932 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6933 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6934 return true;
6935 }
6936 if (K.consume_front("maxntid") && isXYZ(K)) {
6937 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6938 return true;
6939 }
6940 if (K.consume_front("reqntid") && isXYZ(K)) {
6941 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6942 return true;
6943 }
6944 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6945 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6946 return true;
6947 }
6948 if (K == "grid_constant") {
6949 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6950 for (const auto &Op : cast<MDNode>(V)->operands()) {
6951 // For some reason, the index is 1-based in the metadata. Good thing we're
6952 // able to auto-upgrade it!
6953 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6954 cast<Function>(GV)->addParamAttr(Index, Attr);
6955 }
6956 return true;
6957 }
6958
6959 return false;
6960}
6961
6962void llvm::UpgradeNVVMAnnotations(Module &M) {
6963 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6964 if (!NamedMD)
6965 return;
6966
6967 SmallVector<MDNode *, 8> NewNodes;
6968 SmallPtrSet<const MDNode *, 8> SeenNodes;
6969 for (MDNode *MD : NamedMD->operands()) {
6970 if (!SeenNodes.insert(MD).second)
6971 continue;
6972
6973 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6974 if (!GV)
6975 continue;
6976
6977 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6978
6979 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6980 // Each nvvm.annotations metadata entry will be of the following form:
6981 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6982 // start index = 1, to skip the global variable key
6983 // increment = 2, to skip the value for each property-value pairs
6984 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6985 MDString *K = cast<MDString>(MD->getOperand(j));
6986 const MDOperand &V = MD->getOperand(j + 1);
6987 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6988 if (!Upgraded)
6989 NewOperands.append({K, V});
6990 }
6991
6992 if (NewOperands.size() > 1)
6993 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6994 }
6995
6996 NamedMD->clearOperands();
6997 for (MDNode *N : NewNodes)
6998 NamedMD->addOperand(N);
6999}
7000
7001/// This checks for objc retain release marker which should be upgraded. It
7002/// returns true if module is modified.
7003static bool upgradeRetainReleaseMarker(Module &M) {
7004 bool Changed = false;
7005 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
7006 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
7007 if (ModRetainReleaseMarker) {
7008 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
7009 if (Op) {
7010 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
7011 if (ID) {
7012 SmallVector<StringRef, 4> ValueComp;
7013 ID->getString().split(ValueComp, "#");
7014 if (ValueComp.size() == 2) {
7015 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
7016 ID = MDString::get(M.getContext(), NewValue);
7017 }
7018 M.addModuleFlag(Module::Error, MarkerKey, ID);
7019 M.eraseNamedMetadata(ModRetainReleaseMarker);
7020 Changed = true;
7021 }
7022 }
7023 }
7024 return Changed;
7025}
7026
7027void llvm::UpgradeARCRuntime(Module &M) {
7028 // This lambda converts normal function calls to ARC runtime functions to
7029 // intrinsic calls.
7030 auto UpgradeToIntrinsic = [&](const char *OldFunc,
7031 llvm::Intrinsic::ID IntrinsicFunc) {
7032 Function *Fn = M.getFunction(OldFunc);
7033
7034 if (!Fn)
7035 return;
7036
7037 Function *NewFn =
7038 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
7039
7040 for (User *U : make_early_inc_range(Fn->users())) {
7041 CallInst *CI = dyn_cast<CallInst>(U);
7042 if (!CI || CI->getCalledFunction() != Fn)
7043 continue;
7044
7045 IRBuilder<> Builder(CI->getIterator());
7046 FunctionType *NewFuncTy = NewFn->getFunctionType();
7047 SmallVector<Value *, 2> Args;
7048
7049 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7050 // value to the return type of the old function.
7051 if (NewFuncTy->getReturnType() != CI->getType() &&
7052 !CastInst::castIsValid(Instruction::BitCast, CI,
7053 NewFuncTy->getReturnType()))
7054 continue;
7055
7056 bool InvalidCast = false;
7057
7058 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7059 Value *Arg = CI->getArgOperand(I);
7060
7061 // Bitcast argument to the parameter type of the new function if it's
7062 // not a variadic argument.
7063 if (I < NewFuncTy->getNumParams()) {
7064 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7065 // to the parameter type of the new function.
7066 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
7067 NewFuncTy->getParamType(I))) {
7068 InvalidCast = true;
7069 break;
7070 }
7071 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
7072 }
7073 Args.push_back(Arg);
7074 }
7075
7076 if (InvalidCast)
7077 continue;
7078
7079 // Create a call instruction that calls the new function.
7080 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7081 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
7082 NewCall->takeName(CI);
7083
7084 // Bitcast the return value back to the type of the old call.
7085 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
7086
7087 if (!CI->use_empty())
7088 CI->replaceAllUsesWith(NewRetVal);
7089 CI->eraseFromParent();
7090 }
7091
7092 if (Fn->use_empty())
7093 Fn->eraseFromParent();
7094 };
7095
7096 // Unconditionally convert a call to "clang.arc.use" to a call to
7097 // "llvm.objc.clang.arc.use".
7098 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7099
7100 // Upgrade the retain release marker. If there is no need to upgrade
7101 // the marker, that means either the module is already new enough to contain
7102 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7103 if (!upgradeRetainReleaseMarker(M))
7104 return;
7105
7106 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7107 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7108 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7109 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7110 {"objc_autoreleaseReturnValue",
7111 llvm::Intrinsic::objc_autoreleaseReturnValue},
7112 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7113 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7114 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7115 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7116 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7117 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7118 {"objc_release", llvm::Intrinsic::objc_release},
7119 {"objc_retain", llvm::Intrinsic::objc_retain},
7120 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7121 {"objc_retainAutoreleaseReturnValue",
7122 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7123 {"objc_retainAutoreleasedReturnValue",
7124 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7125 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7126 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7127 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7128 {"objc_unsafeClaimAutoreleasedReturnValue",
7129 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7130 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7131 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7132 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7133 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7134 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7135 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7136 {"objc_arc_annotation_topdown_bbstart",
7137 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7138 {"objc_arc_annotation_topdown_bbend",
7139 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7140 {"objc_arc_annotation_bottomup_bbstart",
7141 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7142 {"objc_arc_annotation_bottomup_bbend",
7143 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7144
7145 for (auto &I : RuntimeFuncs)
7146 UpgradeToIntrinsic(I.first, I.second);
7147}
7148
7149// Upgrade the way signing of pointers to init/fini functions is described.
7150//
7151// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7152// constants, if signing was requested. After the upgrade, these arrays contain
7153// plain function pointers and the desired signing schema is described via a
7154// pair of module flags.
7155//
7156// Note that the upgrade is only performed if all elements of *both* arrays
7157// agree on a common signing schema.
7158static bool upgradePtrauthInitFiniArrays(Module &M) {
7159 // As we cannot always decide whether the particular module should have
7160 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7161 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7162 // returns as soon as it spots any non-signed init/fini pointer: either we
7163 // should request non-signed pointers (safe to omit both flags) or there is
7164 // no common schema (and thus we do not modify anything).
7165 //
7166 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7167 // or whether we should request address diversity in addition to the basic
7168 // constant diversity. There is no value representing "decided not to sign"
7169 // for the reasons explained above.
7170 std::optional<bool> UseAddressDisc;
7171
7172 // Do not attempt upgrading if the new module flags already exist.
7173 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7174 for (const MDNode *Flag : ModFlags->operands()) {
7175 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7176 if (Name && (*Name == "ptrauth-init-fini" ||
7177 *Name == "ptrauth-init-fini-address-discrimination"))
7178 return false;
7179 }
7180 }
7181
7182 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7183 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7184 constexpr unsigned ExpectedAddressMarker = 1;
7185
7186 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7187 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7188 return nullptr; // Nothing to upgrade or unknown pattern found.
7189
7190 bool HasAddressDisc;
7191 if (!CPA->hasAddressDiscriminator())
7192 HasAddressDisc = false;
7193 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7194 HasAddressDisc = true;
7195 else
7196 return nullptr; // Unknown pattern.
7197
7198 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7199 return nullptr; // Disagreement with the decided mode.
7200
7201 UseAddressDisc = HasAddressDisc;
7202 return CPA->getPointer();
7203 };
7204
7205 // Do not apply any changes until we know the upgrade is non-ambiguous.
7206 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7207 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7208
7209 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7210 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7211 if (!GV || !GV->hasInitializer())
7212 continue; // Skip, but it is okay to upgrade the other variable.
7213
7214 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7215 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7216 return false;
7217
7218 std::vector<Constant *> NewStructors;
7219 NewStructors.reserve(OldStructorsArray->getNumOperands());
7220
7221 for (Use &U : OldStructorsArray->operands()) {
7222 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7223 if (!Structor || Structor->getNumOperands() != 3)
7224 return false;
7225
7226 Constant *Prio = Structor->getOperand(0);
7227 Constant *Func = Structor->getOperand(1);
7228 Constant *Arg = Structor->getOperand(2);
7229
7230 Func = UpgradeSinglePointer(Func);
7231 if (!Func)
7232 return false;
7233
7234 NewStructors.push_back(
7235 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7236 }
7237
7238 Constant *NewInit =
7239 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7240 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7241 }
7242
7243 if (GlobalArraysToUpgrade.empty())
7244 return false;
7245 assert(UseAddressDisc.has_value());
7246
7247 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7248 GV->setInitializer(NewInit);
7249
7250 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7251 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7252 *UseAddressDisc);
7253
7254 return true;
7255}
7256
7257bool llvm::UpgradeModuleFlags(Module &M) {
7258 bool Changed = false;
7259 Changed |= upgradePtrauthInitFiniArrays(M);
7260
7261 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7262 if (!ModFlags)
7263 return Changed;
7264
7265 bool HasObjCFlag = false, HasClassProperties = false;
7266 bool HasSwiftVersionFlag = false;
7267 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7268 uint32_t SwiftABIVersion;
7269 auto Int8Ty = Type::getInt8Ty(M.getContext());
7270 auto Int32Ty = Type::getInt32Ty(M.getContext());
7271
7272 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7273 MDNode *Op = ModFlags->getOperand(I);
7274 if (Op->getNumOperands() != 3)
7275 continue;
7276 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7277 if (!ID)
7278 continue;
7279 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7280 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7281 Type::getInt32Ty(M.getContext()), B)),
7282 MDString::get(M.getContext(), ID->getString()),
7283 Op->getOperand(2)};
7284 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7285 Changed = true;
7286 };
7287
7288 if (ID->getString() == "Objective-C Image Info Version")
7289 HasObjCFlag = true;
7290 if (ID->getString() == "Objective-C Class Properties")
7291 HasClassProperties = true;
7292 // Upgrade PIC from Error/Max to Min.
7293 if (ID->getString() == "PIC Level") {
7294 if (auto *Behavior =
7295 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7296 uint64_t V = Behavior->getLimitedValue();
7297 if (V == Module::Error || V == Module::Max)
7298 SetBehavior(Module::Min);
7299 }
7300 }
7301 // Upgrade "PIE Level" from Error to Max.
7302 if (ID->getString() == "PIE Level")
7303 if (auto *Behavior =
7304 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7305 if (Behavior->getLimitedValue() == Module::Error)
7306 SetBehavior(Module::Max);
7307
7308 // Upgrade branch protection and return address signing module flags. The
7309 // module flag behavior for these fields were Error and now they are Min.
7310 // The one exception is "sign-return-address-harden".
7311 if (ID->getString() == "branch-target-enforcement" ||
7312 (ID->getString().starts_with("sign-return-address") &&
7313 ID->getString() != "sign-return-address-harden")) {
7314 if (auto *Behavior =
7315 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7316 if (Behavior->getLimitedValue() == Module::Error) {
7317 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7318 Metadata *Ops[3] = {
7319 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7320 Op->getOperand(1), Op->getOperand(2)};
7321 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7322 Changed = true;
7323 }
7324 }
7325 }
7326
7327 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7328 // section name so that llvm-lto will not complain about mismatching
7329 // module flags that is functionally the same.
7330 if (ID->getString() == "Objective-C Image Info Section") {
7331 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7332 SmallVector<StringRef, 4> ValueComp;
7333 Value->getString().split(ValueComp, " ");
7334 if (ValueComp.size() != 1) {
7335 std::string NewValue;
7336 for (auto &S : ValueComp)
7337 NewValue += S.str();
7338 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7339 MDString::get(M.getContext(), NewValue)};
7340 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7341 Changed = true;
7342 }
7343 }
7344 }
7345
7346 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7347 // If the higher bits are set, it adds new module flag for swift info.
7348 if (ID->getString() == "Objective-C Garbage Collection") {
7349 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7350 if (Md) {
7351 assert(Md->getValue() && "Expected non-empty metadata");
7352 auto Type = Md->getValue()->getType();
7353 if (Type == Int8Ty)
7354 continue;
7355 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7356 if ((Val & 0xff) != Val) {
7357 HasSwiftVersionFlag = true;
7358 SwiftABIVersion = (Val & 0xff00) >> 8;
7359 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7360 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7361 }
7362 Metadata *Ops[3] = {
7363 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7364 Op->getOperand(1),
7365 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7366 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7367 Changed = true;
7368 }
7369 }
7370
7371 if (ID->getString() == "amdgpu_code_object_version") {
7372 Metadata *Ops[3] = {
7373 Op->getOperand(0),
7374 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7375 Op->getOperand(2)};
7376 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7377 Changed = true;
7378 }
7379
7380 // clang/PowerPC used to use "float-abi" to describe the long double format;
7381 // it has been renamed to "long-double-type", with its values changed to the
7382 // corresponding IR floating-point type names.
7383 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7384 StringRef Format;
7385 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7386 Format = S->getString();
7387
7388 // The "float-abi" key is now reserved for the target-independent
7389 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7390 // (including unrecognized ones, which were never valid) to the default.
7391 if (!FloatABI::parseABIType(Format)) {
7392 LongDoubleFormat NewFormat =
7393 StringSwitch<LongDoubleFormat>(Format)
7394 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7395 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7396 .Default(LongDoubleFormat::PPCDoubleDouble);
7397 Metadata *Ops[3] = {
7398 Op->getOperand(0),
7399 MDString::get(M.getContext(), "long-double-type"),
7400 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7401 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7402 Changed = true;
7403 }
7404 }
7405 }
7406
7407 // "Objective-C Class Properties" is recently added for Objective-C. We
7408 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7409 // flag of value 0, so we can correclty downgrade this flag when trying to
7410 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7411 // this module flag.
7412 if (HasObjCFlag && !HasClassProperties) {
7413 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7414 (uint32_t)0);
7415 Changed = true;
7416 }
7417
7418 if (HasSwiftVersionFlag) {
7419 M.addModuleFlag(Module::Error, "Swift ABI Version",
7420 SwiftABIVersion);
7421 M.addModuleFlag(Module::Error, "Swift Major Version",
7422 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7423 M.addModuleFlag(Module::Error, "Swift Minor Version",
7424 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7425 Changed = true;
7426 }
7427
7428 return Changed;
7429}
7430
7431bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7432 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7433 // If this metadata has operands, we expect all of them to be either from
7434 // before or from after the format change handled here, so we can bail out
7435 // fast if the first (if any) operands is of the new format.
7436 auto MatchesVersion = [](const MDNode *Op) {
7437 return Op->getNumOperands() >= 3 &&
7438 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7439 cast<ConstantAsMetadata>(Op->getOperand(2))
7440 ->getType()
7441 ->isIntegerTy(64);
7442 };
7443
7444 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7445 MatchesVersion(CFIConsts->getOperand(0)))
7446 return false;
7447
7448 bool Changed = false;
7449 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7450 MDNode *Op = CFIConsts->getOperand(I);
7451 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7452 assert(Op->getNumOperands() >= 2 &&
7453 "Expected at least 2 operands - name and linkage type");
7454 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7455 StringRef Name = NameMD->getString();
7456 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7457 GlobalValue::dropLLVMManglingEscape(Name));
7458
7459 SmallVector<Metadata *, 4> Elts;
7460 Elts.push_back(Op->getOperand(0));
7461 Elts.push_back(Op->getOperand(1));
7462 Elts.push_back(ConstantAsMetadata::get(
7463 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7464
7465 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7466 Elts.push_back(Op->getOperand(J));
7467
7468 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7469 Changed = true;
7470 }
7471
7472 return Changed;
7473}
7474
7475void llvm::UpgradeSectionAttributes(Module &M) {
7476 auto TrimSpaces = [](StringRef Section) -> std::string {
7477 SmallVector<StringRef, 5> Components;
7478 Section.split(Components, ',');
7479
7480 SmallString<32> Buffer;
7481 raw_svector_ostream OS(Buffer);
7482
7483 for (auto Component : Components)
7484 OS << ',' << Component.trim();
7485
7486 return std::string(OS.str().substr(1));
7487 };
7488
7489 for (auto &GV : M.globals()) {
7490 if (!GV.hasSection())
7491 continue;
7492
7493 StringRef Section = GV.getSection();
7494
7495 if (!Section.starts_with("__DATA, __objc_catlist"))
7496 continue;
7497
7498 // __DATA, __objc_catlist, regular, no_dead_strip
7499 // __DATA,__objc_catlist,regular,no_dead_strip
7500 GV.setSection(TrimSpaces(Section));
7501 }
7502}
7503
7504namespace {
7505// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7506// callsites within a function that did not also have the strictfp attribute.
7507// Since 10.0, if strict FP semantics are needed within a function, the
7508// function must have the strictfp attribute and all calls within the function
7509// must also have the strictfp attribute. This latter restriction is
7510// necessary to prevent unwanted libcall simplification when a function is
7511// being cloned (such as for inlining).
7512//
7513// The "dangling" strictfp attribute usage was only used to prevent constant
7514// folding and other libcall simplification. The nobuiltin attribute on the
7515// callsite has the same effect.
7516struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7517 StrictFPUpgradeVisitor() = default;
7518
7519 void visitCallBase(CallBase &Call) {
7520 if (!Call.isStrictFP())
7521 return;
7522 if (isa<ConstrainedFPIntrinsic>(&Call))
7523 return;
7524 // If we get here, the caller doesn't have the strictfp attribute
7525 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7526 Call.removeFnAttr(Attribute::StrictFP);
7527 Call.addFnAttr(Attribute::NoBuiltin);
7528 }
7529};
7530
7531/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7532struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7533 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7534 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7535
7536 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7537 if (!RMW.isFloatingPointOperation())
7538 return;
7539
7540 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7541 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7542 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7543 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7544 }
7545};
7546} // namespace
7547
7548void llvm::UpgradeFunctionAttributes(Function &F) {
7549 // If a function definition doesn't have the strictfp attribute,
7550 // convert any callsite strictfp attributes to nobuiltin.
7551 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7552 StrictFPUpgradeVisitor SFPV;
7553 SFPV.visit(F);
7554 }
7555
7556 // Remove all incompatibile attributes from function.
7557 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7558 F.getReturnType(), F.getAttributes().getRetAttrs()));
7559 for (auto &Arg : F.args())
7560 Arg.removeAttrs(
7561 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7562
7563 bool AddingAttrs = false, RemovingAttrs = false;
7564 AttrBuilder AttrsToAdd(F.getContext());
7565 AttributeMask AttrsToRemove;
7566
7567 // Older versions of LLVM treated an "implicit-section-name" attribute
7568 // similarly to directly setting the section on a Function.
7569 if (Attribute A = F.getFnAttribute("implicit-section-name");
7570 A.isValid() && A.isStringAttribute()) {
7571 F.setSection(A.getValueAsString());
7572 AttrsToRemove.addAttribute("implicit-section-name");
7573 RemovingAttrs = true;
7574 }
7575
7576 if (Attribute A = F.getFnAttribute("nooutline");
7577 A.isValid() && A.isStringAttribute()) {
7578 AttrsToRemove.addAttribute("nooutline");
7579 AttrsToAdd.addAttribute(Attribute::NoOutline);
7580 AddingAttrs = RemovingAttrs = true;
7581 }
7582
7583 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7584 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7585 AttrsToRemove.addAttribute("uniform-work-group-size");
7586 RemovingAttrs = true;
7587 if (A.getValueAsString() == "true") {
7588 AttrsToAdd.addAttribute("uniform-work-group-size");
7589 AddingAttrs = true;
7590 }
7591 }
7592
7593 if (!F.empty()) {
7594 // For some reason this is called twice, and the first time is before any
7595 // instructions are loaded into the body.
7596
7597 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7598 A.isValid()) {
7599
7600 if (A.getValueAsBool()) {
7601 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7602 Visitor.visit(F);
7603 }
7604
7605 // We will leave behind dead attribute uses on external declarations, but
7606 // clang never added these to declarations anyway.
7607 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7608 RemovingAttrs = true;
7609 }
7610 }
7611
7612 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7613 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7614
7615 bool HandleDenormalMode = false;
7616
7617 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7618 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7619 if (ParsedMode.isValid()) {
7620 DenormalFPMath = ParsedMode;
7621 AttrsToRemove.addAttribute("denormal-fp-math");
7622 AddingAttrs = RemovingAttrs = true;
7623 HandleDenormalMode = true;
7624 }
7625 }
7626
7627 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7628 Attr.isValid()) {
7629 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7630 if (ParsedMode.isValid()) {
7631 DenormalFPMathF32 = ParsedMode;
7632 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7633 AddingAttrs = RemovingAttrs = true;
7634 HandleDenormalMode = true;
7635 }
7636 }
7637
7638 if (HandleDenormalMode)
7639 AttrsToAdd.addDenormalFPEnvAttr(
7640 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7641
7642 if (RemovingAttrs)
7643 F.removeFnAttrs(AttrsToRemove);
7644
7645 if (AddingAttrs)
7646 F.addFnAttrs(AttrsToAdd);
7647}
7648
7649// Check if the function attribute is not present and set it.
7650static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7651 StringRef Value) {
7652 if (!F.hasFnAttribute(FnAttrName)) {
7653 F.addFnAttr(FnAttrName, Value);
7654 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7655 << "\", function: " << F.getName() << "\n");
7656 }
7657}
7658
7659// Check if the function attribute is not present and set it if needed.
7660// If the attribute is "false" then removes it.
7661// If the attribute is "true" resets it to a valueless attribute.
7662static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7663 if (!F.hasFnAttribute(FnAttrName)) {
7664 if (Set) {
7665 F.addFnAttr(FnAttrName);
7666 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7667 << ", function: " << F.getName() << "\n");
7668 }
7669 } else {
7670 auto A = F.getFnAttribute(FnAttrName);
7671 if ("false" == A.getValueAsString()) {
7672 F.removeFnAttr(FnAttrName);
7673 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7674 << "=\"false\", function: " << F.getName() << "\n");
7675 } else if ("true" == A.getValueAsString()) {
7676 F.removeFnAttr(FnAttrName);
7677 F.addFnAttr(FnAttrName);
7678 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7679 << "=\"true\", function: " << F.getName() << "\n");
7680 }
7681 }
7682}
7683
7684static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7685 StringRef Key, uint32_t Val) {
7686 M.setModuleFlag(Behavior, Key, Val);
7687 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7688 << Key << ", " << Val << "}\n");
7689}
7690
7691void llvm::copyModuleAttrToFunctions(Module &M) {
7692 Triple T(M.getTargetTriple());
7693 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7694 return;
7695
7696 uint64_t BTEValue = 0;
7697 uint64_t BPPLRValue = 0;
7698 uint64_t GCSValue = 0;
7699 uint64_t SRAValue = 0;
7700 uint64_t SRAALLValue = 0;
7701 uint64_t SRABKeyValue = 0;
7702
7703 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7704 if (ModFlags) {
7705 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7706 MDNode *Op = ModFlags->getOperand(I);
7707 if (Op->getNumOperands() != 3)
7708 continue;
7709
7710 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7711 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7712 if (!ID || !CI)
7713 continue;
7714
7715 StringRef IDStr = ID->getString();
7716 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7717 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7718 : IDStr == "guarded-control-stack" ? &GCSValue
7719 : IDStr == "sign-return-address" ? &SRAValue
7720 : IDStr == "sign-return-address-all" ? &SRAALLValue
7721 : IDStr == "sign-return-address-with-bkey"
7722 ? &SRABKeyValue
7723 : nullptr;
7724 if (!ValPtr)
7725 continue;
7726
7727 *ValPtr = CI->getZExtValue();
7728 if (*ValPtr == 2)
7729 return;
7730
7731 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7732 << ")\n");
7733 }
7734 }
7735
7736 bool BTE = BTEValue == 1;
7737 bool BPPLR = BPPLRValue == 1;
7738 bool GCS = GCSValue == 1;
7739 bool SRA = SRAValue == 1;
7740
7741 StringRef SignTypeValue = "non-leaf";
7742 if (SRA && SRAALLValue == 1)
7743 SignTypeValue = "all";
7744
7745 StringRef SignKeyValue = "a_key";
7746 if (SRA && SRABKeyValue == 1)
7747 SignKeyValue = "b_key";
7748
7749 for (Function &F : M.getFunctionList()) {
7750 if (F.isDeclaration())
7751 continue;
7752
7753 if (SRA) {
7754 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7755 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7756 } else {
7757 if (auto A = F.getFnAttribute("sign-return-address");
7758 A.isValid() && "none" == A.getValueAsString()) {
7759 F.removeFnAttr("sign-return-address");
7760 F.removeFnAttr("sign-return-address-key");
7761 }
7762 }
7763 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7764 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7765 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7766 }
7767
7768 if (BTE)
7769 ConvertModuleFlag(M, llvm::Module::Min, "branch-target-enforcement", 2);
7770 if (BPPLR)
7771 ConvertModuleFlag(M, llvm::Module::Min, "branch-protection-pauth-lr", 2);
7772 if (GCS)
7773 ConvertModuleFlag(M, llvm::Module::Min, "guarded-control-stack", 2);
7774 if (SRA) {
7775 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address", 2);
7776 if (SRAALLValue == 1)
7777 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-all", 2);
7778 if (SRABKeyValue == 1) {
7779 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-with-bkey",
7780 2);
7781 }
7782 }
7783}
7784
7785/// Return the replacement tags if \p T still uses a removed two-operand form.
7786static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7787 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7788 return nullptr;
7789 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7790 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7791}
7792
7793/// Build the single-operand node that replaces a boolean operand: nonzero
7794/// selects the enable tag, zero the disable tag.
7795static Metadata *makeBooleanLoopNode(LLVMContext &C,
7796 const BooleanLoopTags &Tags,
7797 const MDOperand &Op) {
7798 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7799 return MDTuple::get(C,
7800 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7801}
7802
7803static bool isOldLoopArgument(Metadata *MD) {
7804 auto *T = dyn_cast_or_null<MDTuple>(MD);
7805 if (!T)
7806 return false;
7807 if (T->getNumOperands() < 1)
7808 return false;
7809 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7810 if (!S)
7811 return false;
7812 if (S->getString().starts_with("llvm.vectorizer."))
7813 return true;
7814 return getOldBooleanLoopTags(T) != nullptr;
7815}
7816
7817static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7818 StringRef OldPrefix = "llvm.vectorizer.";
7819 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7820
7821 if (OldTag == "llvm.vectorizer.unroll")
7822 return MDString::get(C, "llvm.loop.interleave.count");
7823
7824 return MDString::get(
7825 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7826 .str());
7827}
7828
7829static Metadata *upgradeLoopArgument(Metadata *MD) {
7830 auto *T = dyn_cast_or_null<MDTuple>(MD);
7831 if (!T)
7832 return MD;
7833 if (T->getNumOperands() < 1)
7834 return MD;
7835 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7836 if (!OldTag)
7837 return MD;
7838
7839 LLVMContext &C = T->getContext();
7840
7841 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7842 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7843 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7844
7845 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7846 return MD;
7847
7848 // This has an old tag. Upgrade it.
7849 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7850
7851 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7852 // vectorize.enable/disable pair, not a two-operand enable node.
7853 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7854 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7855 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7856
7857 SmallVector<Metadata *, 8> Ops;
7858 Ops.reserve(T->getNumOperands());
7859 Ops.push_back(NewTag);
7860 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7861 Ops.push_back(T->getOperand(I));
7862
7863 return MDTuple::get(C, Ops);
7864}
7865
7866MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7867 auto *T = dyn_cast<MDTuple>(&N);
7868 if (!T)
7869 return &N;
7870
7871 if (none_of(T->operands(), isOldLoopArgument))
7872 return &N;
7873
7874 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7875 // any MDNode carrying those tags with more than one operand, so a leftover
7876 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7877 // In-place mutation is safe on distinct MDNodes.
7878 if (T->isDistinct()) {
7879 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7880 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7881 if (OpT && getOldBooleanLoopTags(OpT))
7882 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7883 }
7884 if (none_of(T->operands(), isOldLoopArgument))
7885 return &N;
7886 }
7887
7888 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7889 // attachment; the original distinct loop-ID is kept as the first operand.
7890 SmallVector<Metadata *, 8> Ops;
7891 Ops.reserve(T->getNumOperands());
7892 for (Metadata *MD : T->operands())
7893 Ops.push_back(upgradeLoopArgument(MD));
7894
7895 return MDTuple::get(T->getContext(), Ops);
7896}
7897
7898std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7899 Triple T(TT);
7900 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7901 // the address space of globals to 1. This does not apply to SPIRV Logical.
7902 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7903 !DL.contains("-G") && !DL.starts_with("G")) {
7904 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7905 }
7906
7907 if (T.isLoongArch64() || T.isRISCV64()) {
7908 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7909 auto I = DL.find("-n64-");
7910 if (I != StringRef::npos)
7911 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7912 return DL.str();
7913 }
7914
7915 // AMDGPU data layout upgrades.
7916 std::string Res = DL.str();
7917 if (T.isAMDGPU()) {
7918 // Define address spaces for constants.
7919 if (!DL.contains("-G") && !DL.starts_with("G"))
7920 Res.append(Res.empty() ? "G1" : "-G1");
7921
7922 // AMDGCN data layout upgrades.
7923 if (T.isAMDGCN()) {
7924
7925 // Add missing non-integral declarations.
7926 // This goes before adding new address spaces to prevent incoherent string
7927 // values.
7928 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7929 Res.append("-ni:7:8:9");
7930 // Update ni:7 to ni:7:8:9.
7931 if (DL.ends_with("ni:7"))
7932 Res.append(":8:9");
7933 if (DL.ends_with("ni:7:8"))
7934 Res.append(":9");
7935
7936 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7937 // resources) An empty data layout has already been upgraded to G1 by now.
7938 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7939 Res.append("-p7:160:256:256:32");
7940 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7941 Res.append("-p8:128:128:128:48");
7942 constexpr StringRef OldP8("-p8:128:128-");
7943 if (DL.contains(OldP8))
7944 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7945 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7946 Res.append("-p9:192:256:256:32");
7947
7948 // Add sizing for address space 10 through 15.
7949 // AS 10-14 are reserved and defaulted to 32:32
7950 // AS 15 is in use and is 32:32.
7951 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7952 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7953 Res.append(("-" + AS + ":32:32").str());
7954 }
7955 }
7956
7957 // Upgrade the ELF mangling mode.
7958 if (!DL.contains("m:e"))
7959 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7960
7961 return Res;
7962 }
7963
7964 if (T.isSystemZ() && !DL.empty()) {
7965 // Make sure the stack alignment is present.
7966 if (!DL.contains("-S64"))
7967 return "E-S64" + DL.drop_front(1).str();
7968 return DL.str();
7969 }
7970
7971 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7972 // If the datalayout matches the expected format, add pointer size address
7973 // spaces to the datalayout.
7974 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7975 if (!DL.contains(AddrSpaces)) {
7976 SmallVector<StringRef, 4> Groups;
7977 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7978 if (R.match(Res, &Groups))
7979 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7980 }
7981 };
7982
7983 // AArch64 data layout upgrades.
7984 if (T.isAArch64()) {
7985 // Add "-Fn32"
7986 if (!DL.empty() && !DL.contains("-Fn32"))
7987 Res.append("-Fn32");
7988 AddPtr32Ptr64AddrSpaces();
7989 return Res;
7990 }
7991
7992 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7993 T.isWasm()) {
7994 // Mips64 with o32 ABI did not add "-i128:128".
7995 // Add "-i128:128"
7996 std::string I64 = "-i64:64";
7997 std::string I128 = "-i128:128";
7998 if (!StringRef(Res).contains(I128)) {
7999 size_t Pos = Res.find(I64);
8000 if (Pos != size_t(-1))
8001 Res.insert(Pos + I64.size(), I128);
8002 }
8003 }
8004
8005 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
8006 size_t Pos = Res.find("-S128");
8007 if (Pos == StringRef::npos)
8008 Pos = Res.size();
8009 Res.insert(Pos, "-f64:32:64");
8010 }
8011
8012 // ARM data layout upgrades.
8013 // Add -Fi8 if a -F has not already been specified.
8014 if (T.isARM() && !DL.empty() && !DL.contains("Fi") && !DL.contains("Fn")) {
8015 const StringRef p3232 = "p:32:32";
8016 size_t Pos = Res.find(p3232);
8017 if (Pos != StringRef::npos)
8018 Res.insert(Pos + p3232.size(), "-Fi8");
8019 }
8020
8021 if (!T.isX86())
8022 return Res;
8023
8024 AddPtr32Ptr64AddrSpaces();
8025
8026 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
8027 // for i128 operations prior to this being reflected in the data layout, and
8028 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
8029 // boundaries, so although this is a breaking change, the upgrade is expected
8030 // to fix more IR than it breaks.
8031 // Intel MCU is an exception and uses 4-byte-alignment.
8032 if (!T.isOSIAMCU()) {
8033 std::string I128 = "-i128:128";
8034 if (StringRef Ref = Res; !Ref.contains(I128)) {
8035 SmallVector<StringRef, 4> Groups;
8036 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8037 if (R.match(Res, &Groups))
8038 Res = (Groups[1] + I128 + Groups[3]).str();
8039 }
8040 }
8041
8042 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
8043 // Raising the alignment is safe because Clang did not produce f80 values in
8044 // the MSVC environment before this upgrade was added.
8045 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8046 StringRef Ref = Res;
8047 auto I = Ref.find("-f80:32-");
8048 if (I != StringRef::npos)
8049 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
8050 }
8051
8052 return Res;
8053}
8054
8055void llvm::UpgradeAttributes(AttrBuilder &B) {
8056 StringRef FramePointer;
8057 Attribute A = B.getAttribute("no-frame-pointer-elim");
8058 if (A.isValid()) {
8059 // The value can be "true" or "false".
8060 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8061 B.removeAttribute("no-frame-pointer-elim");
8062 }
8063 if (B.contains("no-frame-pointer-elim-non-leaf")) {
8064 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8065 if (FramePointer != "all")
8066 FramePointer = "non-leaf";
8067 B.removeAttribute("no-frame-pointer-elim-non-leaf");
8068 }
8069 if (!FramePointer.empty())
8070 B.addAttribute("frame-pointer", FramePointer);
8071
8072 A = B.getAttribute("null-pointer-is-valid");
8073 if (A.isValid()) {
8074 // The value can be "true" or "false".
8075 bool NullPointerIsValid = A.getValueAsString() == "true";
8076 B.removeAttribute("null-pointer-is-valid");
8077 if (NullPointerIsValid)
8078 B.addAttribute(Attribute::NullPointerIsValid);
8079 }
8080
8081 A = B.getAttribute("uniform-work-group-size");
8082 if (A.isValid()) {
8083 StringRef Val = A.getValueAsString();
8084 if (!Val.empty()) {
8085 bool IsTrue = Val == "true";
8086 B.removeAttribute("uniform-work-group-size");
8087 if (IsTrue)
8088 B.addAttribute("uniform-work-group-size");
8089 }
8090 }
8091}
8092
8093void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8094 // clang.arc.attachedcall bundles are now required to have an operand.
8095 // If they don't, it's okay to drop them entirely: when there is an operand,
8096 // the "attachedcall" is meaningful and required, but without an operand,
8097 // it's just a marker NOP. Dropping it merely prevents an optimization.
8098 erase_if(Bundles, [&](OperandBundleDef &OBD) {
8099 return OBD.getTag() == "clang.arc.attachedcall" &&
8100 OBD.inputs().empty();
8101 });
8102}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
Imm
uint64_t Imm
Definition AMDGPUDisassembler.cpp:552
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2718
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1499
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
Definition AutoUpgrade.cpp:1705
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7829
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1587
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6897
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1741
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2558
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1372
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5736
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1339
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1352
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:7003
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2819
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3045
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1684
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2793
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:125
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1637
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7650
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1441
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6901
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5596
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7817
isValidLegacyNVVMFPArithIntrinsic
static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
Definition AutoUpgrade.cpp:1537
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5696
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6864
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:112
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2673
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2930
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1680
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2644
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5233
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3031
NVVMFMAIIDs
static constexpr Intrinsic::ID NVVMFMAIIDs[2][3]
Definition AutoUpgrade.cpp:1530
NVVMFMulIIDs
static constexpr Intrinsic::ID NVVMFMulIIDs[2][3]
Definition AutoUpgrade.cpp:1526
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7786
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:556
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2970
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1421
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6810
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1594
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7795
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:179
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2592
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1600
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1219
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1396
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7803
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5303
ConvertModuleFlag
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
Definition AutoUpgrade.cpp:7684
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:96
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5666
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5393
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2909
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5608
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:159
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:794
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5767
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2624
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2656
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1725
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2858
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:67
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:80
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:169
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1194
upgradeNVVMFPArithCall
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:3294
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2992
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2778
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3320
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1295
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:3024
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:3054
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5618
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7662
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2940
getNVVMFPArithUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:1554
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:71
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2891
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1323
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1262
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5679
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:143
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7158
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3535
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5714
NVVMFAddIIDs
static constexpr Intrinsic::ID NVVMFAddIIDs[2][3]
Definition AutoUpgrade.cpp:1522
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:274
T1
#define T1
Definition Mips16ISelLowering.cpp:273
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
C1
int64_t C1
Definition RISCVLateBranchOpt.cpp:64
CC
RISCVCC::CondCode CC
Definition RISCVLateBranchOpt.cpp:57
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:480
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:774
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:934
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:786
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:790
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:811
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:850
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:794
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:800
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:854
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:842
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:802
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:822
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:818
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:846
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:981
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:508
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:356
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1595
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1715
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3198
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3481
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1830
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1454
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2521
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2453
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2507
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:2034
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1521
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1685
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:44
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:449
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:389
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3621
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2445
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:728
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:565
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2909
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1790
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:697
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1522
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:320
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::op_end
op_iterator op_end() const
Definition Metadata.h:1431
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::op_begin
op_iterator op_begin() const
Definition Metadata.h:1427
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:615
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:597
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1496
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1525
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:107
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1797
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1594
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1586
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1582
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1601
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1893
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1592
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2092
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2095
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:594
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:720
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:425
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1261
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::find
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringRef::consume_front
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:194
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5351
llvm::Value::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Definition Metadata.cpp:1672
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:390
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:549
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:708
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:315
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1451
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:637
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1423
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1416
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5828
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7475
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3285
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:201
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2507
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7866
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:8055
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6701
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6962
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7257
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7431
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7691
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:8093
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6790
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7898
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::UpgradeTBAAStructNode
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
Definition AutoUpgrade.cpp:6746
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2526
llvm::Key
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Definition PassManager.h:695
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:716
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:134
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6768
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getIROptions
const IROptions & getIROptions(const LLVMContext &C)
Definition IROptions.h:27
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6820
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7548
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6720
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:7027
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:8155
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:120
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:122
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:121
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
Generated on for LLVM by doxygen 1.14.0