LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstVisitor.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsAArch64.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsARM.h"
39#include "llvm/IR/IntrinsicsNVPTX.h"
40#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/IR/IntrinsicsWebAssembly.h"
42#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/MDBuilder.h"
45#include "llvm/IR/Metadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/NVVMIntrinsicUtils.h"
48#include "llvm/IR/Value.h"
49#include "llvm/IR/Verifier.h"
50#include "llvm/Support/AMDGPUAddrSpace.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/NVPTXAddrSpace.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/Regex.h"
57#include "llvm/Support/TimeProfiler.h"
58#include "llvm/TargetParser/Triple.h"
59#include <cstdint>
60#include <cstring>
61#include <numeric>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "auto-upgrade"
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(0);
85 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(F);
90 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(F);
106 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(F);
119 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
134 return false;
135
136 rename(F);
137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
152 return false;
153
154 rename(F);
155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(F);
165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(F);
175 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front("avx."))
186 return (Name.starts_with("blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with("movnt.") || // Added in 3.2
191 Name.starts_with("sqrt.p") || // Added in 7.0
192 Name.starts_with("storeu.") || // Added in 3.9
193 Name.starts_with("vbroadcast.s") || // Added in 3.5
194 Name.starts_with("vbroadcastf128") || // Added in 4.0
195 Name.starts_with("vextractf128.") || // Added in 3.7
196 Name.starts_with("vinsertf128.") || // Added in 3.7
197 Name.starts_with("vperm2f128.") || // Added in 6.0
198 Name.starts_with("vpermil.")); // Added in 3.1
199
200 if (Name.consume_front("avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with("pabs.") || // Added in 6.0
203 Name.starts_with("padds.") || // Added in 8.0
204 Name.starts_with("paddus.") || // Added in 8.0
205 Name.starts_with("pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with("pbroadcast") || // Added in 3.8
208 Name.starts_with("pcmpeq.") || // Added in 3.1
209 Name.starts_with("pcmpgt.") || // Added in 3.1
210 Name.starts_with("pmax") || // Added in 3.9
211 Name.starts_with("pmin") || // Added in 3.9
212 Name.starts_with("pmovsx") || // Added in 3.9
213 Name.starts_with("pmovzx") || // Added in 3.9
214 Name.starts_with("pmulh.w") || // Added in 24.0
215 Name.starts_with("pmulhu.w") || // Added in 24.0
216 Name == "pmul.dq" || // Added in 7.0
217 Name == "pmulu.dq" || // Added in 7.0
218 Name.starts_with("psll.dq") || // Added in 3.7
219 Name.starts_with("psrl.dq") || // Added in 3.7
220 Name.starts_with("psubs.") || // Added in 8.0
221 Name.starts_with("psubus.") || // Added in 8.0
222 Name.starts_with("vbroadcast") || // Added in 3.8
223 Name == "vbroadcasti128" || // Added in 3.7
224 Name == "vextracti128" || // Added in 3.7
225 Name == "vinserti128" || // Added in 3.7
226 Name == "vperm2i128"); // Added in 6.0
227
228 if (Name.consume_front("avx512.")) {
229 if (Name.consume_front("mask."))
230 // 'avx512.mask.*'
231 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
232 Name.starts_with("and.") || // Added in 3.9
233 Name.starts_with("andn.") || // Added in 3.9
234 Name.starts_with("broadcast.s") || // Added in 3.9
235 Name.starts_with("broadcastf32x4.") || // Added in 6.0
236 Name.starts_with("broadcastf32x8.") || // Added in 6.0
237 Name.starts_with("broadcastf64x2.") || // Added in 6.0
238 Name.starts_with("broadcastf64x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x4.") || // Added in 6.0
240 Name.starts_with("broadcasti32x8.") || // Added in 6.0
241 Name.starts_with("broadcasti64x2.") || // Added in 6.0
242 Name.starts_with("broadcasti64x4.") || // Added in 6.0
243 Name.starts_with("cmp.b") || // Added in 5.0
244 Name.starts_with("cmp.d") || // Added in 5.0
245 Name.starts_with("cmp.q") || // Added in 5.0
246 Name.starts_with("cmp.w") || // Added in 5.0
247 Name.starts_with("compress.b") || // Added in 9.0
248 Name.starts_with("compress.d") || // Added in 9.0
249 Name.starts_with("compress.p") || // Added in 9.0
250 Name.starts_with("compress.q") || // Added in 9.0
251 Name.starts_with("compress.store.") || // Added in 7.0
252 Name.starts_with("compress.w") || // Added in 9.0
253 Name.starts_with("conflict.") || // Added in 9.0
254 Name.starts_with("cvtdq2pd.") || // Added in 4.0
255 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
256 Name == "cvtpd2dq.256" || // Added in 7.0
257 Name == "cvtpd2ps.256" || // Added in 7.0
258 Name == "cvtps2pd.128" || // Added in 7.0
259 Name == "cvtps2pd.256" || // Added in 7.0
260 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
261 Name == "cvtqq2ps.256" || // Added in 9.0
262 Name == "cvtqq2ps.512" || // Added in 9.0
263 Name == "cvttpd2dq.256" || // Added in 7.0
264 Name == "cvttps2dq.128" || // Added in 7.0
265 Name == "cvttps2dq.256" || // Added in 7.0
266 Name.starts_with("cvtudq2pd.") || // Added in 4.0
267 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
268 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
269 Name == "cvtuqq2ps.256" || // Added in 9.0
270 Name == "cvtuqq2ps.512" || // Added in 9.0
271 Name.starts_with("dbpsadbw.") || // Added in 7.0
272 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
273 Name.starts_with("expand.b") || // Added in 9.0
274 Name.starts_with("expand.d") || // Added in 9.0
275 Name.starts_with("expand.load.") || // Added in 7.0
276 Name.starts_with("expand.p") || // Added in 9.0
277 Name.starts_with("expand.q") || // Added in 9.0
278 Name.starts_with("expand.w") || // Added in 9.0
279 Name.starts_with("fpclass.p") || // Added in 7.0
280 Name.starts_with("insert") || // Added in 4.0
281 Name.starts_with("load.") || // Added in 3.9
282 Name.starts_with("loadu.") || // Added in 3.9
283 Name.starts_with("lzcnt.") || // Added in 5.0
284 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with("movddup") || // Added in 3.9
287 Name.starts_with("move.s") || // Added in 4.0
288 Name.starts_with("movshdup") || // Added in 3.9
289 Name.starts_with("movsldup") || // Added in 3.9
290 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
291 Name.starts_with("or.") || // Added in 3.9
292 Name.starts_with("pabs.") || // Added in 6.0
293 Name.starts_with("packssdw.") || // Added in 5.0
294 Name.starts_with("packsswb.") || // Added in 5.0
295 Name.starts_with("packusdw.") || // Added in 5.0
296 Name.starts_with("packuswb.") || // Added in 5.0
297 Name.starts_with("padd.") || // Added in 4.0
298 Name.starts_with("padds.") || // Added in 8.0
299 Name.starts_with("paddus.") || // Added in 8.0
300 Name.starts_with("palignr.") || // Added in 3.9
301 Name.starts_with("pand.") || // Added in 3.9
302 Name.starts_with("pandn.") || // Added in 3.9
303 Name.starts_with("pavg") || // Added in 6.0
304 Name.starts_with("pbroadcast") || // Added in 6.0
305 Name.starts_with("pcmpeq.") || // Added in 3.9
306 Name.starts_with("pcmpgt.") || // Added in 3.9
307 Name.starts_with("perm.df.") || // Added in 3.9
308 Name.starts_with("perm.di.") || // Added in 3.9
309 Name.starts_with("permvar.") || // Added in 7.0
310 Name.starts_with("pmaddubs.w.") || // Added in 7.0
311 Name.starts_with("pmaddw.d.") || // Added in 7.0
312 Name.starts_with("pmax") || // Added in 4.0
313 Name.starts_with("pmin") || // Added in 4.0
314 Name == "pmov.qd.256" || // Added in 9.0
315 Name == "pmov.qd.512" || // Added in 9.0
316 Name == "pmov.wb.256" || // Added in 9.0
317 Name == "pmov.wb.512" || // Added in 9.0
318 Name.starts_with("pmovsx") || // Added in 4.0
319 Name.starts_with("pmovzx") || // Added in 4.0
320 Name.starts_with("pmul.dq.") || // Added in 4.0
321 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
322 Name.starts_with("pmulh.w.") || // Added in 7.0
323 Name.starts_with("pmulhu.w.") || // Added in 7.0
324 Name.starts_with("pmull.") || // Added in 4.0
325 Name.starts_with("pmultishift.qb.") || // Added in 8.0
326 Name.starts_with("pmulu.dq.") || // Added in 4.0
327 Name.starts_with("por.") || // Added in 3.9
328 Name.starts_with("prol.") || // Added in 8.0
329 Name.starts_with("prolv.") || // Added in 8.0
330 Name.starts_with("pror.") || // Added in 8.0
331 Name.starts_with("prorv.") || // Added in 8.0
332 Name.starts_with("pshuf.b.") || // Added in 4.0
333 Name.starts_with("pshuf.d.") || // Added in 3.9
334 Name.starts_with("pshufh.w.") || // Added in 3.9
335 Name.starts_with("pshufl.w.") || // Added in 3.9
336 Name.starts_with("psll.d") || // Added in 4.0
337 Name.starts_with("psll.q") || // Added in 4.0
338 Name.starts_with("psll.w") || // Added in 4.0
339 Name.starts_with("pslli") || // Added in 4.0
340 Name.starts_with("psllv") || // Added in 4.0
341 Name.starts_with("psra.d") || // Added in 4.0
342 Name.starts_with("psra.q") || // Added in 4.0
343 Name.starts_with("psra.w") || // Added in 4.0
344 Name.starts_with("psrai") || // Added in 4.0
345 Name.starts_with("psrav") || // Added in 4.0
346 Name.starts_with("psrl.d") || // Added in 4.0
347 Name.starts_with("psrl.q") || // Added in 4.0
348 Name.starts_with("psrl.w") || // Added in 4.0
349 Name.starts_with("psrli") || // Added in 4.0
350 Name.starts_with("psrlv") || // Added in 4.0
351 Name.starts_with("psub.") || // Added in 4.0
352 Name.starts_with("psubs.") || // Added in 8.0
353 Name.starts_with("psubus.") || // Added in 8.0
354 Name.starts_with("pternlog.") || // Added in 7.0
355 Name.starts_with("punpckh") || // Added in 3.9
356 Name.starts_with("punpckl") || // Added in 3.9
357 Name.starts_with("pxor.") || // Added in 3.9
358 Name.starts_with("shuf.f") || // Added in 6.0
359 Name.starts_with("shuf.i") || // Added in 6.0
360 Name.starts_with("shuf.p") || // Added in 4.0
361 Name.starts_with("sqrt.p") || // Added in 7.0
362 Name.starts_with("store.b.") || // Added in 3.9
363 Name.starts_with("store.d.") || // Added in 3.9
364 Name.starts_with("store.p") || // Added in 3.9
365 Name.starts_with("store.q.") || // Added in 3.9
366 Name.starts_with("store.w.") || // Added in 3.9
367 Name == "store.ss" || // Added in 7.0
368 Name.starts_with("storeu.") || // Added in 3.9
369 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
370 Name.starts_with("ucmp.") || // Added in 5.0
371 Name.starts_with("unpckh.") || // Added in 3.9
372 Name.starts_with("unpckl.") || // Added in 3.9
373 Name.starts_with("valign.") || // Added in 4.0
374 Name == "vcvtph2ps.128" || // Added in 11.0
375 Name == "vcvtph2ps.256" || // Added in 11.0
376 Name.starts_with("vextract") || // Added in 4.0
377 Name.starts_with("vfmadd.") || // Added in 7.0
378 Name.starts_with("vfmaddsub.") || // Added in 7.0
379 Name.starts_with("vfnmadd.") || // Added in 7.0
380 Name.starts_with("vfnmsub.") || // Added in 7.0
381 Name.starts_with("vpdpbusd.") || // Added in 7.0
382 Name.starts_with("vpdpbusds.") || // Added in 7.0
383 Name.starts_with("vpdpwssd.") || // Added in 7.0
384 Name.starts_with("vpdpwssds.") || // Added in 7.0
385 Name.starts_with("vpermi2var.") || // Added in 7.0
386 Name.starts_with("vpermil.p") || // Added in 3.9
387 Name.starts_with("vpermilvar.") || // Added in 4.0
388 Name.starts_with("vpermt2var.") || // Added in 7.0
389 Name.starts_with("vpmadd52") || // Added in 7.0
390 Name.starts_with("vpshld.") || // Added in 7.0
391 Name.starts_with("vpshldv.") || // Added in 8.0
392 Name.starts_with("vpshrd.") || // Added in 7.0
393 Name.starts_with("vpshrdv.") || // Added in 8.0
394 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
395 Name.starts_with("xor.")); // Added in 3.9
396
397 if (Name.consume_front("mask3."))
398 // 'avx512.mask3.*'
399 return (Name.starts_with("vfmadd.") || // Added in 7.0
400 Name.starts_with("vfmaddsub.") || // Added in 7.0
401 Name.starts_with("vfmsub.") || // Added in 7.0
402 Name.starts_with("vfmsubadd.") || // Added in 7.0
403 Name.starts_with("vfnmsub.")); // Added in 7.0
404
405 if (Name.consume_front("maskz."))
406 // 'avx512.maskz.*'
407 return (Name.starts_with("pternlog.") || // Added in 7.0
408 Name.starts_with("vfmadd.") || // Added in 7.0
409 Name.starts_with("vfmaddsub.") || // Added in 7.0
410 Name.starts_with("vpdpbusd.") || // Added in 7.0
411 Name.starts_with("vpdpbusds.") || // Added in 7.0
412 Name.starts_with("vpdpwssd.") || // Added in 7.0
413 Name.starts_with("vpdpwssds.") || // Added in 7.0
414 Name.starts_with("vpermt2var.") || // Added in 7.0
415 Name.starts_with("vpmadd52") || // Added in 7.0
416 Name.starts_with("vpshldv.") || // Added in 8.0
417 Name.starts_with("vpshrdv.")); // Added in 8.0
418
419 // 'avx512.*'
420 return (Name == "movntdqa" || // Added in 5.0
421 Name == "pmul.dq.512" || // Added in 7.0
422 Name == "pmulu.dq.512" || // Added in 7.0
423 Name.starts_with("broadcastm") || // Added in 6.0
424 Name.starts_with("cmp.p") || // Added in 12.0
425 Name.starts_with("cvtb2mask.") || // Added in 7.0
426 Name.starts_with("cvtd2mask.") || // Added in 7.0
427 Name.starts_with("cvtmask2") || // Added in 5.0
428 Name.starts_with("cvtq2mask.") || // Added in 7.0
429 Name == "cvtusi2sd" || // Added in 7.0
430 Name.starts_with("cvtw2mask.") || // Added in 7.0
431 Name == "kand.w" || // Added in 7.0
432 Name == "kandn.w" || // Added in 7.0
433 Name == "knot.w" || // Added in 7.0
434 Name == "kor.w" || // Added in 7.0
435 Name == "kortestc.w" || // Added in 7.0
436 Name == "kortestz.w" || // Added in 7.0
437 Name.starts_with("kunpck") || // added in 6.0
438 Name == "kxnor.w" || // Added in 7.0
439 Name == "kxor.w" || // Added in 7.0
440 Name.starts_with("padds.") || // Added in 8.0
441 Name.starts_with("pbroadcast") || // Added in 3.9
442 Name.starts_with("pmulh.w") || // Added in 24.0
443 Name.starts_with("pmulhu.w") || // Added in 24.0
444 Name.starts_with("prol") || // Added in 8.0
445 Name.starts_with("pror") || // Added in 8.0
446 Name.starts_with("psll.dq") || // Added in 3.9
447 Name.starts_with("psrl.dq") || // Added in 3.9
448 Name.starts_with("psubs.") || // Added in 8.0
449 Name.starts_with("ptestm") || // Added in 6.0
450 Name.starts_with("ptestnm") || // Added in 6.0
451 Name.starts_with("storent.") || // Added in 3.9
452 Name.starts_with("vbroadcast.s") || // Added in 7.0
453 Name.starts_with("vpshld.") || // Added in 8.0
454 Name.starts_with("vpshrd.")); // Added in 8.0
455 }
456
457 if (Name.consume_front("fma."))
458 return (Name.starts_with("vfmadd.") || // Added in 7.0
459 Name.starts_with("vfmsub.") || // Added in 7.0
460 Name.starts_with("vfmsubadd.") || // Added in 7.0
461 Name.starts_with("vfnmadd.") || // Added in 7.0
462 Name.starts_with("vfnmsub.")); // Added in 7.0
463
464 if (Name.consume_front("fma4."))
465 return Name.starts_with("vfmadd.s"); // Added in 7.0
466
467 if (Name.consume_front("sse."))
468 return (Name == "add.ss" || // Added in 4.0
469 Name == "cvtsi2ss" || // Added in 7.0
470 Name == "cvtsi642ss" || // Added in 7.0
471 Name == "div.ss" || // Added in 4.0
472 Name == "mul.ss" || // Added in 4.0
473 Name.starts_with("sqrt.p") || // Added in 7.0
474 Name == "sqrt.ss" || // Added in 7.0
475 Name.starts_with("storeu.") || // Added in 3.9
476 Name == "sub.ss"); // Added in 4.0
477
478 if (Name.consume_front("sse2."))
479 return (Name == "add.sd" || // Added in 4.0
480 Name == "cvtdq2pd" || // Added in 3.9
481 Name == "cvtdq2ps" || // Added in 7.0
482 Name == "cvtps2pd" || // Added in 3.9
483 Name == "cvtsi2sd" || // Added in 7.0
484 Name == "cvtsi642sd" || // Added in 7.0
485 Name == "cvtss2sd" || // Added in 7.0
486 Name == "div.sd" || // Added in 4.0
487 Name == "mul.sd" || // Added in 4.0
488 Name.starts_with("padds.") || // Added in 8.0
489 Name.starts_with("paddus.") || // Added in 8.0
490 Name.starts_with("pcmpeq.") || // Added in 3.1
491 Name.starts_with("pcmpgt.") || // Added in 3.1
492 Name == "pmaxs.w" || // Added in 3.9
493 Name == "pmaxu.b" || // Added in 3.9
494 Name == "pmins.w" || // Added in 3.9
495 Name == "pminu.b" || // Added in 3.9
496 Name == "pmulh.w" || // Added in 24.0
497 Name == "pmulhu.w" || // Added in 24.0
498 Name == "pmulu.dq" || // Added in 7.0
499 Name.starts_with("pshuf") || // Added in 3.9
500 Name.starts_with("psll.dq") || // Added in 3.7
501 Name.starts_with("psrl.dq") || // Added in 3.7
502 Name.starts_with("psubs.") || // Added in 8.0
503 Name.starts_with("psubus.") || // Added in 8.0
504 Name.starts_with("sqrt.p") || // Added in 7.0
505 Name == "sqrt.sd" || // Added in 7.0
506 Name == "storel.dq" || // Added in 3.9
507 Name.starts_with("storeu.") || // Added in 3.9
508 Name == "sub.sd"); // Added in 4.0
509
510 if (Name.consume_front("sse41."))
511 return (Name.starts_with("blendp") || // Added in 3.7
512 Name == "movntdqa" || // Added in 5.0
513 Name == "pblendw" || // Added in 3.7
514 Name == "pmaxsb" || // Added in 3.9
515 Name == "pmaxsd" || // Added in 3.9
516 Name == "pmaxud" || // Added in 3.9
517 Name == "pmaxuw" || // Added in 3.9
518 Name == "pminsb" || // Added in 3.9
519 Name == "pminsd" || // Added in 3.9
520 Name == "pminud" || // Added in 3.9
521 Name == "pminuw" || // Added in 3.9
522 Name.starts_with("pmovsx") || // Added in 3.8
523 Name.starts_with("pmovzx") || // Added in 3.9
524 Name == "pmuldq"); // Added in 7.0
525
526 if (Name.consume_front("sse42."))
527 return Name == "crc32.64.8"; // Added in 3.4
528
529 if (Name.consume_front("sse4a."))
530 return Name.starts_with("movnt."); // Added in 3.9
531
532 if (Name.consume_front("ssse3."))
533 return (Name == "pabs.b.128" || // Added in 6.0
534 Name == "pabs.d.128" || // Added in 6.0
535 Name == "pabs.w.128"); // Added in 6.0
536
537 if (Name.consume_front("xop."))
538 return (Name == "vpcmov" || // Added in 3.8
539 Name == "vpcmov.256" || // Added in 5.0
540 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
541 Name.starts_with("vprot")); // Added in 8.0
542
543 if (Name.consume_front("bmi."))
544 return (Name.starts_with("pdep.") || // Added in 23.0
545 Name.starts_with("pext.")); // Added in 23.0
546
547 return (Name == "addcarry.u32" || // Added in 8.0
548 Name == "addcarry.u64" || // Added in 8.0
549 Name == "addcarryx.u32" || // Added in 8.0
550 Name == "addcarryx.u64" || // Added in 8.0
551 Name == "subborrow.u32" || // Added in 8.0
552 Name == "subborrow.u64" || // Added in 8.0
553 Name.starts_with("vcvtph2ps.")); // Added in 11.0
554}
555
556static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
557 Function *&NewFn) {
558 // Only handle intrinsics that start with "x86.".
559 if (!Name.consume_front("x86."))
560 return false;
561
562 if (shouldUpgradeX86Intrinsic(F, Name)) {
563 NewFn = nullptr;
564 return true;
565 }
566
567 if (Name == "rdtscp") { // Added in 8.0
568 // If this intrinsic has 0 operands, it's the new version.
569 if (F->getFunctionType()->getNumParams() == 0)
570 return false;
571
572 rename(F);
573 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
574 Intrinsic::x86_rdtscp);
575 return true;
576 }
577
578 Intrinsic::ID ID;
579
580 // SSE4.1 ptest functions may have an old signature.
581 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
582 ID = StringSwitch<Intrinsic::ID>(Name)
583 .Case("c", Intrinsic::x86_sse41_ptestc)
584 .Case("z", Intrinsic::x86_sse41_ptestz)
585 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
586 .Default(Intrinsic::not_intrinsic);
587 if (ID != Intrinsic::not_intrinsic)
588 return upgradePTESTIntrinsic(F, ID, NewFn);
589
590 return false;
591 }
592
593 // Several blend and other instructions with masks used the wrong number of
594 // bits.
595
596 // Added in 3.6
597 ID = StringSwitch<Intrinsic::ID>(Name)
598 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
599 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
600 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
601 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
602 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
603 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
604 .Default(Intrinsic::not_intrinsic);
605 if (ID != Intrinsic::not_intrinsic)
606 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
607
608 if (Name.consume_front("avx512.")) {
609 if (Name.consume_front("mask.cmp.")) {
610 // Added in 7.0
611 ID = StringSwitch<Intrinsic::ID>(Name)
612 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
613 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
614 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
615 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
616 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
617 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
618 .Default(Intrinsic::not_intrinsic);
619 if (ID != Intrinsic::not_intrinsic)
620 return upgradeX86MaskedFPCompare(F, ID, NewFn);
621 } else if (Name.starts_with("vpdpbusd.") ||
622 Name.starts_with("vpdpbusds.")) {
623 // Added in 21.1
624 ID = StringSwitch<Intrinsic::ID>(Name)
625 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
626 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
627 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
628 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
629 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
630 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
631 .Default(Intrinsic::not_intrinsic);
632 if (ID != Intrinsic::not_intrinsic)
633 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
634 } else if (Name.starts_with("vpdpwssd.") ||
635 Name.starts_with("vpdpwssds.")) {
636 // Added in 21.1
637 ID = StringSwitch<Intrinsic::ID>(Name)
638 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
639 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
640 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
641 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
642 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
643 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
644 .Default(Intrinsic::not_intrinsic);
645 if (ID != Intrinsic::not_intrinsic)
646 return upgradeX86MultiplyAddWords(F, ID, NewFn);
647 }
648 return false; // No other 'x86.avx512.*'.
649 }
650
651 if (Name.consume_front("avx2.")) {
652 if (Name.consume_front("vpdpb")) {
653 // Added in 21.1
654 ID = StringSwitch<Intrinsic::ID>(Name)
655 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
656 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
657 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
658 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
659 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
660 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
661 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
662 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
663 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
664 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
665 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
666 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
667 .Default(Intrinsic::not_intrinsic);
668 if (ID != Intrinsic::not_intrinsic)
669 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
670 } else if (Name.consume_front("vpdpw")) {
671 // Added in 21.1
672 ID = StringSwitch<Intrinsic::ID>(Name)
673 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
674 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
675 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
676 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
677 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
678 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
679 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
680 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
681 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
682 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
683 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
684 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
685 .Default(Intrinsic::not_intrinsic);
686 if (ID != Intrinsic::not_intrinsic)
687 return upgradeX86MultiplyAddWords(F, ID, NewFn);
688 }
689 return false; // No other 'x86.avx2.*'
690 }
691
692 if (Name.consume_front("avx10.")) {
693 if (Name.consume_front("vpdpb")) {
694 // Added in 21.1
695 ID = StringSwitch<Intrinsic::ID>(Name)
696 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
697 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
698 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
699 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
700 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
701 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
702 .Default(Intrinsic::not_intrinsic);
703 if (ID != Intrinsic::not_intrinsic)
704 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
705 } else if (Name.consume_front("vpdpw")) {
706 ID = StringSwitch<Intrinsic::ID>(Name)
707 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
708 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
709 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
710 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
711 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
712 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
713 .Default(Intrinsic::not_intrinsic);
714 if (ID != Intrinsic::not_intrinsic)
715 return upgradeX86MultiplyAddWords(F, ID, NewFn);
716 }
717 return false; // No other 'x86.avx10.*'
718 }
719
720 if (Name.consume_front("avx512bf16.")) {
721 // Added in 9.0
722 ID = StringSwitch<Intrinsic::ID>(Name)
723 .Case("cvtne2ps2bf16.128",
724 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
725 .Case("cvtne2ps2bf16.256",
726 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
727 .Case("cvtne2ps2bf16.512",
728 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
729 .Case("mask.cvtneps2bf16.128",
730 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
731 .Case("cvtneps2bf16.256",
732 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
733 .Case("cvtneps2bf16.512",
734 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
735 .Default(Intrinsic::not_intrinsic);
736 if (ID != Intrinsic::not_intrinsic)
737 return upgradeX86BF16Intrinsic(F, ID, NewFn);
738
739 // Added in 9.0
740 ID = StringSwitch<Intrinsic::ID>(Name)
741 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
742 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
743 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
744 .Default(Intrinsic::not_intrinsic);
745 if (ID != Intrinsic::not_intrinsic)
746 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
747 return false; // No other 'x86.avx512bf16.*'.
748 }
749
750 if (Name.consume_front("xop.")) {
751 Intrinsic::ID ID = Intrinsic::not_intrinsic;
752 if (Name.starts_with("vpermil2")) { // Added in 3.9
753 // Upgrade any XOP PERMIL2 index operand still using a float/double
754 // vector.
755 auto Idx = F->getFunctionType()->getParamType(2);
756 if (Idx->isFPOrFPVectorTy()) {
757 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
758 unsigned EltSize = Idx->getScalarSizeInBits();
759 if (EltSize == 64 && IdxSize == 128)
760 ID = Intrinsic::x86_xop_vpermil2pd;
761 else if (EltSize == 32 && IdxSize == 128)
762 ID = Intrinsic::x86_xop_vpermil2ps;
763 else if (EltSize == 64 && IdxSize == 256)
764 ID = Intrinsic::x86_xop_vpermil2pd_256;
765 else
766 ID = Intrinsic::x86_xop_vpermil2ps_256;
767 }
768 } else if (F->arg_size() == 2)
769 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
770 ID = StringSwitch<Intrinsic::ID>(Name)
771 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
772 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
773 .Default(Intrinsic::not_intrinsic);
774
775 if (ID != Intrinsic::not_intrinsic) {
776 rename(F);
777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
778 return true;
779 }
780 return false; // No other 'x86.xop.*'
781 }
782
783 if (Name == "seh.recoverfp") {
784 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
785 Intrinsic::eh_recoverfp);
786 return true;
787 }
788
789 return false;
790}
791
792// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
793// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
794static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
795 StringRef Name,
796 Function *&NewFn) {
797 if (Name.starts_with("rbit")) {
798 // '(arm|aarch64).rbit'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
801 return true;
802 }
803
804 if (Name == "thread.pointer") {
805 // '(arm|aarch64).thread.pointer'.
806 NewFn = Intrinsic::getOrInsertDeclaration(
807 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
808 return true;
809 }
810
811 bool Neon = Name.consume_front("neon.");
812 if (Neon) {
813 // '(arm|aarch64).neon.*'.
814 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
815 // v16i8 respectively.
816 if (Name.consume_front("bfdot.")) {
817 // (arm|aarch64).neon.bfdot.*'.
818 Intrinsic::ID ID =
819 StringSwitch<Intrinsic::ID>(Name)
820 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
821 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
822 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
823 .Default(Intrinsic::not_intrinsic);
824 if (ID != Intrinsic::not_intrinsic) {
825 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
826 assert((OperandWidth == 64 || OperandWidth == 128) &&
827 "Unexpected operand width");
828 LLVMContext &Ctx = F->getParent()->getContext();
829 std::array<Type *, 2> Tys{
830 {F->getReturnType(),
831 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
832 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
833 return true;
834 }
835 return false; // No other '(arm|aarch64).neon.bfdot.*'.
836 }
837
838 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
839 // anymore and accept v8bf16 instead of v16i8.
840 if (Name.consume_front("bfm")) {
841 // (arm|aarch64).neon.bfm*'.
842 if (Name.consume_back(".v4f32.v16i8")) {
843 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
844 Intrinsic::ID ID =
845 StringSwitch<Intrinsic::ID>(Name)
846 .Case("mla",
847 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
849 .Case("lalb",
850 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
851 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
852 .Case("lalt",
853 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
854 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
855 .Default(Intrinsic::not_intrinsic);
856 if (ID != Intrinsic::not_intrinsic) {
857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
858 return true;
859 }
860 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
861 }
862 return false; // No other '(arm|aarch64).neon.bfm*.
863 }
864 // Continue on to Aarch64 Neon or Arm Neon.
865 }
866 // Continue on to Arm or Aarch64.
867
868 if (IsArm) {
869 // 'arm.*'.
870 if (Neon) {
871 // 'arm.neon.*'.
872 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
873 .StartsWith("vclz.", Intrinsic::ctlz)
874 .StartsWith("vcnt.", Intrinsic::ctpop)
875 .StartsWith("vqadds.", Intrinsic::sadd_sat)
876 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
877 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
878 .StartsWith("vqsubu.", Intrinsic::usub_sat)
879 .StartsWith("vrinta.", Intrinsic::round)
880 .StartsWith("vrintn.", Intrinsic::roundeven)
881 .StartsWith("vrintm.", Intrinsic::floor)
882 .StartsWith("vrintp.", Intrinsic::ceil)
883 .StartsWith("vrintx.", Intrinsic::rint)
884 .StartsWith("vrintz.", Intrinsic::trunc)
885 .Default(Intrinsic::not_intrinsic);
886 if (ID != Intrinsic::not_intrinsic) {
887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
888 F->arg_begin()->getType());
889 return true;
890 }
891
892 if (Name.consume_front("vst")) {
893 // 'arm.neon.vst*'.
894 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
895 SmallVector<StringRef, 2> Groups;
896 if (vstRegex.match(Name, &Groups)) {
897 static const Intrinsic::ID StoreInts[] = {
898 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
899 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
900
901 static const Intrinsic::ID StoreLaneInts[] = {
902 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
903 Intrinsic::arm_neon_vst4lane};
904
905 auto fArgs = F->getFunctionType()->params();
906 Type *Tys[] = {fArgs[0], fArgs[1]};
907 if (Groups[1].size() == 1)
908 NewFn = Intrinsic::getOrInsertDeclaration(
909 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
910 else
911 NewFn = Intrinsic::getOrInsertDeclaration(
912 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
913 return true;
914 }
915 return false; // No other 'arm.neon.vst*'.
916 }
917
918 return false; // No other 'arm.neon.*'.
919 }
920
921 if (Name.consume_front("mve.")) {
922 // 'arm.mve.*'.
923 if (Name == "vctp64") {
924 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
925 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
926 // the function and deal with it below in UpgradeIntrinsicCall.
927 rename(F);
928 return true;
929 }
930 return false; // Not 'arm.mve.vctp64'.
931 }
932
933 if (Name.starts_with("vrintn.v")) {
934 NewFn = Intrinsic::getOrInsertDeclaration(
935 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
936 return true;
937 }
938
939 // These too are changed to accept a v2i1 instead of the old v4i1.
940 if (Name.consume_back(".v4i1")) {
941 // 'arm.mve.*.v4i1'.
942 if (Name.consume_back(".predicated.v2i64.v4i32"))
943 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
944 return Name == "mull.int" || Name == "vqdmull";
945
946 if (Name.consume_back(".v2i64")) {
947 // 'arm.mve.*.v2i64.v4i1'
948 bool IsGather = Name.consume_front("vldr.gather.");
949 if (IsGather || Name.consume_front("vstr.scatter.")) {
950 if (Name.consume_front("base.")) {
951 // Optional 'wb.' prefix.
952 Name.consume_front("wb.");
953 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
954 // predicated.v2i64.v2i64.v4i1'.
955 return Name == "predicated.v2i64";
956 }
957
958 if (Name.consume_front("offset.predicated."))
959 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
960 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
961
962 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
963 return false;
964 }
965
966 return false; // No other 'arm.mve.*.v2i64.v4i1'.
967 }
968 return false; // No other 'arm.mve.*.v4i1'.
969 }
970 return false; // No other 'arm.mve.*'.
971 }
972
973 if (Name.consume_front("cde.vcx")) {
974 // 'arm.cde.vcx*'.
975 if (Name.consume_back(".predicated.v2i64.v4i1"))
976 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
977 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
978 Name == "3q" || Name == "3qa";
979
980 return false; // No other 'arm.cde.vcx*'.
981 }
982 } else {
983 // 'aarch64.*'.
984 if (Neon) {
985 // 'aarch64.neon.*'.
986 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
987 .StartsWith("frintn", Intrinsic::roundeven)
988 .StartsWith("rbit", Intrinsic::bitreverse)
989 .Default(Intrinsic::not_intrinsic);
990 if (ID != Intrinsic::not_intrinsic) {
991 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
992 F->arg_begin()->getType());
993 return true;
994 }
995
996 Intrinsic::ID MinMaxID =
997 StringSwitch<Intrinsic::ID>(Name.split('.').first)
998 .Case("smax", Intrinsic::smax)
999 .Case("smin", Intrinsic::smin)
1000 .Case("umax", Intrinsic::umax)
1001 .Case("umin", Intrinsic::umin)
1002 .Default(Intrinsic::not_intrinsic);
1003 if (MinMaxID != Intrinsic::not_intrinsic) {
1004 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1005 return false; // Invalid IR.
1006 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1007 F->getReturnType());
1008 return true;
1009 }
1010
1011 if (Name.starts_with("addp")) {
1012 // 'aarch64.neon.addp*'.
1013 if (F->arg_size() != 2)
1014 return false; // Invalid IR.
1015 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1016 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1017 NewFn = Intrinsic::getOrInsertDeclaration(
1018 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1019 return true;
1020 }
1021 }
1022
1023 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1024 if (Name.starts_with("bfcvt")) {
1025 NewFn = nullptr;
1026 return true;
1027 }
1028
1029 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1030 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1031 NewFn = nullptr;
1032 return true;
1033 }
1034
1035 return false; // No other 'aarch64.neon.*'.
1036 }
1037 if (Name.consume_front("sve.")) {
1038 // 'aarch64.sve.*'.
1039 if (Name.consume_front("bf")) {
1040 if (Name == "mmla") {
1041 Type *Tys[] = {F->getReturnType(),
1042 std::next(F->arg_begin())->getType()};
1043 NewFn = Intrinsic::getOrInsertDeclaration(
1044 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1045 return true;
1046 }
1047 if (Name.consume_back(".lane")) {
1048 // 'aarch64.sve.bf*.lane'.
1049 Intrinsic::ID ID =
1050 StringSwitch<Intrinsic::ID>(Name)
1051 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1052 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1053 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1054 .Default(Intrinsic::not_intrinsic);
1055 if (ID != Intrinsic::not_intrinsic) {
1056 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1057 return true;
1058 }
1059 return false; // No other 'aarch64.sve.bf*.lane'.
1060 }
1061 return false; // No other 'aarch64.sve.bf*'.
1062 }
1063
1064 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1065 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1066 NewFn = nullptr;
1067 return true;
1068 }
1069
1070 if (Name.consume_front("convert.from.svbool")) {
1071 // 'aarch64.sve.convert.from.svbool'
1072 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1073 if (!TTy || TTy->getName() != "aarch64.svcount")
1074 return false;
1075
1076 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1077 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1078 return true;
1079 }
1080
1081 if (Name.consume_front("convert.to.svbool")) {
1082 // 'aarch64.sve.convert.to.svbool'
1083 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1084 if (!TTy || TTy->getName() != "aarch64.svcount")
1085 return false;
1086
1087 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1088 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1089 return true;
1090 }
1091
1092 if (Name.consume_front("addqv")) {
1093 // 'aarch64.sve.addqv'.
1094 if (!F->getReturnType()->isFPOrFPVectorTy())
1095 return false;
1096
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {F->getReturnType(), Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1101 return true;
1102 }
1103
1104 if (Name.consume_front("ld")) {
1105 // 'aarch64.sve.ld*'.
1106 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1107 if (LdRegex.match(Name)) {
1108 Type *ScalarTy =
1109 cast<VectorType>(F->getReturnType())->getElementType();
1110 ElementCount EC =
1111 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1112 assert(F->arg_size() == 2 &&
1113 "Expected 2 arguments for ld* intrinsic.");
1114 Type *PtrTy = F->getArg(1)->getType();
1115 Type *Ty = VectorType::get(ScalarTy, EC);
1116 static const Intrinsic::ID LoadIDs[] = {
1117 Intrinsic::aarch64_sve_ld2_sret,
1118 Intrinsic::aarch64_sve_ld3_sret,
1119 Intrinsic::aarch64_sve_ld4_sret,
1120 };
1121 NewFn = Intrinsic::getOrInsertDeclaration(
1122 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1123 return true;
1124 }
1125 return false; // No other 'aarch64.sve.ld*'.
1126 }
1127
1128 if (Name.consume_front("tuple.")) {
1129 // 'aarch64.sve.tuple.*'.
1130 if (Name.starts_with("get")) {
1131 // 'aarch64.sve.tuple.get*'.
1132 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1133 NewFn = Intrinsic::getOrInsertDeclaration(
1134 F->getParent(), Intrinsic::vector_extract, Tys);
1135 return true;
1136 }
1137
1138 if (Name.starts_with("set")) {
1139 // 'aarch64.sve.tuple.set*'.
1140 auto Args = F->getFunctionType()->params();
1141 Type *Tys[] = {Args[0], Args[2], Args[1]};
1142 NewFn = Intrinsic::getOrInsertDeclaration(
1143 F->getParent(), Intrinsic::vector_insert, Tys);
1144 return true;
1145 }
1146
1147 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1148 if (CreateTupleRegex.match(Name)) {
1149 // 'aarch64.sve.tuple.create*'.
1150 auto Args = F->getFunctionType()->params();
1151 Type *Tys[] = {F->getReturnType(), Args[1]};
1152 NewFn = Intrinsic::getOrInsertDeclaration(
1153 F->getParent(), Intrinsic::vector_insert, Tys);
1154 return true;
1155 }
1156 return false; // No other 'aarch64.sve.tuple.*'.
1157 }
1158
1159 if (Name.starts_with("rev.nxv")) {
1160 // 'aarch64.sve.rev.<Ty>'
1161 NewFn = Intrinsic::getOrInsertDeclaration(
1162 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1163 return true;
1164 }
1165
1166 return false; // No other 'aarch64.sve.*'.
1167 }
1168 if (Name.consume_front("sme.")) {
1169 // 'aarch64.sme.*'.
1170 if (Name.consume_front("ftmopa.")) {
1171 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1172 // intrinsics to model their FPMR dependency.
1173 Intrinsic::ID ID =
1174 StringSwitch<Intrinsic::ID>(Name)
1175 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1176 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1177 .Default(Intrinsic::not_intrinsic);
1178 if (ID != Intrinsic::not_intrinsic) {
1179 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1180 return true;
1181 }
1182 return false; // No other 'aarch64.sme.ftmopa.*'.
1183 }
1184
1185 return false; // No other 'aarch64.sme.*'.
1186 }
1187 }
1188 return false; // No other 'arm.*', 'aarch64.*'.
1189}
1190
1191// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1192// declarations requiring an auto-upgrade. The same set applies to the
1193// cluster (g2s) and CTA (g2s_cta) variants.
1194#define NVVM_TMA_G2S_MODES(M) \
1195 M(tile_1d, "tile.1d") \
1196 M(tile_2d, "tile.2d") \
1197 M(tile_3d, "tile.3d") \
1198 M(tile_4d, "tile.4d") \
1199 M(tile_5d, "tile.5d") \
1200 M(tile_gather4_2d, "tile.gather4.2d") \
1201 M(im2col_3d, "im2col.3d") \
1202 M(im2col_4d, "im2col.4d") \
1203 M(im2col_5d, "im2col.5d") \
1204 M(im2col_w_3d, "im2col.w.3d") \
1205 M(im2col_w_4d, "im2col.w.4d") \
1206 M(im2col_w_5d, "im2col.w.5d") \
1207 M(im2col_w_128_3d, "im2col.w.128.3d") \
1208 M(im2col_w_128_4d, "im2col.w.128.4d") \
1209 M(im2col_w_128_5d, "im2col.w.128.5d")
1210
1211// Two legacy tails are:
1212//
1213// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1215//
1216// The current tail appends a trailing i32 %flag_valid_pattern, so both
1217// legacy tails are recognized by an i1 at parameter N-2.
1218static Intrinsic::ID
1219shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1220 SmallVectorImpl<Type *> &OvlTys) {
1221 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1222 return Intrinsic::not_intrinsic;
1223
1224#define G2S_ID(ID_SUFFIX, NAME) \
1225 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1226 // clang-format off
1227 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1228 NVVM_TMA_G2S_MODES(G2S_ID)
1229 .Default(Intrinsic::not_intrinsic);
1230#undef G2S_ID
1231 // clang-format on
1232 if (ID == Intrinsic::not_intrinsic)
1233 return ID;
1234
1235 size_t NumParams = F->getFunctionType()->getNumParams();
1236
1237 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1238 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1239 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1240 return Intrinsic::not_intrinsic;
1241
1242 // The multicast mask is the parameter immediately before the i64
1243 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1244 ArrayRef<Type *> Params = F->getFunctionType()->params();
1245 size_t MaskIdx =
1246 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1247 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1248 "expected the i64 cache-hint after the multicast mask");
1249 Type *MaskTy = Params[MaskIdx];
1250 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1251 OvlTys.push_back(MaskTy);
1252
1253 return ID;
1254}
1255
1256// The legacy tail is:
1257//
1258// arg1, arg2, .. i64 %ch, i1 %flag_ch
1259//
1260// The current tail appends a trailing i32 %flag_valid_pattern, so the
1261// legacy tail is recognized by an i1 at parameter N-1.
1262static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1263 StringRef Name) {
1264 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1265 return Intrinsic::not_intrinsic;
1266
1267#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1268 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1269 // clang-format off
1270 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1271 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1272 .Default(Intrinsic::not_intrinsic);
1273#undef G2S_CTA_ID
1274 // clang-format on
1275 if (ID == Intrinsic::not_intrinsic)
1276 return ID;
1277
1278 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1279 // with i32 %flag_valid_pattern, for which N-1 is i32.
1280 if (!F->getFunctionType()
1281 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1282 ->isIntegerTy(1))
1283 return Intrinsic::not_intrinsic;
1284
1285 return ID;
1286}
1287// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1288//
1289// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1290//
1291// The current intrinsic is overloaded on the multicast-mask type and takes a
1292// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1293// parameter N-1.
1294static Intrinsic::ID
1295shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1296 SmallVectorImpl<Type *> &OvlTys) {
1297 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1298 return Intrinsic::not_intrinsic;
1299
1300 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1301 // i32 %flag_valid_pattern, for which N-1 is i32.
1302 size_t NumParams = F->getFunctionType()->getNumParams();
1303 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1304 return Intrinsic::not_intrinsic;
1305
1306 // The multicast mask is parameter 4; legacy IR only uses i16.
1307 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1308 if (!MaskTy->isIntegerTy(16))
1309 return Intrinsic::not_intrinsic;
1310 OvlTys.push_back(MaskTy);
1311
1312 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1313}
1314
1315// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1316//
1317// ..., i64 %ch, i1 %flag_ch
1318//
1319// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1320// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1321// recognized by an i1 at parameter N-1, whereas the current tail ends
1322// with an i32.
1323static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1324 StringRef Name) {
1325 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1326 return Intrinsic::not_intrinsic;
1327
1328 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1329 // i32 %flag_valid_pattern, for which N-1 is i32.
1330 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1331 return Intrinsic::not_intrinsic;
1332
1333 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1334}
1335
1336// The legacy TMA reduction intrinsics encode the reduction operator in their
1337// name, while the current ones take it as an immediate argument. Map the
1338// operator part of a legacy name to the corresponding immediate value.
1339static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1340 return StringSwitch<std::optional<unsigned>>(Name)
1341 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1342 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1343 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1344 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1345 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1346 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1347 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1348 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1349 .Default(std::nullopt);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1353 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1354 return Intrinsic::not_intrinsic;
1355
1356 auto [RedOpName, ShapeName] = Name.split('.');
1357 if (!getNVPTXTMAReductionOp(RedOpName))
1358 return Intrinsic::not_intrinsic;
1359
1360 return StringSwitch<Intrinsic::ID>(ShapeName)
1361 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1362 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1363 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1364 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1365 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1366 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1367 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1368 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1369 .Default(Intrinsic::not_intrinsic);
1370}
1371
1372static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1373 StringRef Name) {
1374 if (Name.consume_front("mapa.shared.cluster"))
1375 if (F->getReturnType()->getPointerAddressSpace() ==
1376 NVPTXAS::ADDRESS_SPACE_SHARED)
1377 return Intrinsic::nvvm_mapa_shared_cluster;
1378
1379 if (Name.consume_front("cp.async.bulk.")) {
1380 Intrinsic::ID ID =
1381 StringSwitch<Intrinsic::ID>(Name)
1382 .Case("shared.cta.to.cluster",
1383 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1384 .Default(Intrinsic::not_intrinsic);
1385
1386 if (ID != Intrinsic::not_intrinsic)
1387 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1388 NVPTXAS::ADDRESS_SPACE_SHARED)
1389 return ID;
1390 }
1391
1392 return Intrinsic::not_intrinsic;
1393}
1394
1395static Intrinsic::ID
1396shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1397 if (!Name.consume_front("tcgen05.commit."))
1398 return Intrinsic::not_intrinsic;
1399
1400 if (Name.consume_front("shared."))
1401 return StringSwitch<Intrinsic::ID>(Name)
1402 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1403 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1404 .Default(Intrinsic::not_intrinsic);
1405
1406 if (Name.consume_front("mc.shared.")) {
1407 // Only upgrade older i16 mc variants.
1408 if (!F->getArg(1)->getType()->isIntegerTy(16))
1409 return Intrinsic::not_intrinsic;
1410
1411 return StringSwitch<Intrinsic::ID>(Name)
1412 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1413 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1414 .Default(Intrinsic::not_intrinsic);
1415 }
1416
1417 return Intrinsic::not_intrinsic;
1418}
1419
1420static Intrinsic::ID
1421shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1422 if (F->arg_size() != 2)
1423 return Intrinsic::not_intrinsic;
1424
1425 if (Name.consume_front("tcgen05.alloc.shared.") ||
1426 Name.consume_front("tcgen05.alloc."))
1427 return StringSwitch<Intrinsic::ID>(Name)
1428 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1429 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1430 .Default(Intrinsic::not_intrinsic);
1431
1432 if (Name.consume_front("tcgen05.dealloc."))
1433 return StringSwitch<Intrinsic::ID>(Name)
1434 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1435 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1436 .Default(Intrinsic::not_intrinsic);
1437
1438 return Intrinsic::not_intrinsic;
1439}
1440
1441static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1442 if (Name.consume_front("fma.rn."))
1443 return StringSwitch<Intrinsic::ID>(Name)
1444 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1445 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1446 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1447 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1448 .Default(Intrinsic::not_intrinsic);
1449
1450 if (Name.consume_front("fmax."))
1451 return StringSwitch<Intrinsic::ID>(Name)
1452 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1453 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1454 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1455 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1456 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1457 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1458 .Case("ftz.nan.xorsign.abs.bf16",
1459 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1460 .Case("ftz.nan.xorsign.abs.bf16x2",
1461 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1462 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1463 .Case("ftz.xorsign.abs.bf16x2",
1464 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1465 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1466 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1467 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1468 .Case("nan.xorsign.abs.bf16x2",
1469 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1470 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1471 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1472 .Default(Intrinsic::not_intrinsic);
1473
1474 if (Name.consume_front("fmin."))
1475 return StringSwitch<Intrinsic::ID>(Name)
1476 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1477 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1478 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1479 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1480 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1481 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1482 .Case("ftz.nan.xorsign.abs.bf16",
1483 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1484 .Case("ftz.nan.xorsign.abs.bf16x2",
1485 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1486 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1487 .Case("ftz.xorsign.abs.bf16x2",
1488 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1489 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1490 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1491 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1492 .Case("nan.xorsign.abs.bf16x2",
1493 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1494 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1495 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1496 .Default(Intrinsic::not_intrinsic);
1497
1498 if (Name.consume_front("neg."))
1499 return StringSwitch<Intrinsic::ID>(Name)
1500 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1501 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1502 .Default(Intrinsic::not_intrinsic);
1503
1504 return Intrinsic::not_intrinsic;
1505}
1506
1507static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1508 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1509 FunctionType *OldFnTy = F->getFunctionType();
1510 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1511 return OldTy->getScalarType()->isIntegerTy() &&
1512 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1513 };
1514
1515 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1516 return false;
1517
1518 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1519 return false;
1520
1521 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1522 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1523 return false;
1524
1525 return true;
1526}
1527
1528// Overloaded fadd/fmul intrinsic IDs, indexed by [`.ftz`][`.sat`].
1529static constexpr Intrinsic::ID NVVMFAddIIDs[2][2] = {
1530 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat},
1531 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat}};
1532static constexpr Intrinsic::ID NVVMFMulIIDs[2][2] = {
1533 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat},
1534 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat}};
1535
1536static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1537getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][2]) {
1538 auto [Modifiers, Type] = Name.rsplit('.');
1539 if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
1540 return std::nullopt;
1541
1542 std::optional<llvm::RoundingMode> RoundingMode =
1543 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1544 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1545 .Case("rz", llvm::RoundingMode::TowardZero)
1546 .Case("rm", llvm::RoundingMode::TowardNegative)
1547 .Case("rp", llvm::RoundingMode::TowardPositive)
1548 .Default(std::nullopt);
1549 if (!RoundingMode)
1550 return std::nullopt;
1551
1552 StringRef Rest = Modifiers.drop_front(2);
1553 const bool IsFTZ = Rest.consume_front(".ftz");
1554 const bool IsSat = Rest.consume_front(".sat");
1555 if (!Rest.empty())
1556 return std::nullopt;
1557
1558 return std::make_pair(IIDs[IsFTZ][IsSat], *RoundingMode);
1559}
1560
1561static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1562 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1563 return Intrinsic::not_intrinsic;
1564
1565 return Intrinsic::nvvm_mbarrier_init;
1566}
1567
1568static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1569 return Name.consume_front("local") || Name.consume_front("shared") ||
1570 Name.consume_front("global") || Name.consume_front("constant") ||
1571 Name.consume_front("param");
1572}
1573
1574static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1575 if (!Name.consume_front("vp."))
1576 return 0;
1577 return StringSwitch<unsigned>(Name)
1578 .StartsWith("select", Instruction::Select)
1579 .StartsWith("add", Instruction::Add)
1580 .StartsWith("sub", Instruction::Sub)
1581 .StartsWith("mul", Instruction::Mul)
1582 .StartsWith("ashr", Instruction::AShr)
1583 .StartsWith("lshr", Instruction::LShr)
1584 .StartsWith("shl", Instruction::Shl)
1585 .StartsWith("or", Instruction::Or)
1586 .StartsWith("and", Instruction::And)
1587 .StartsWith("xor", Instruction::Xor)
1588 .StartsWith("fadd", Instruction::FAdd)
1589 .StartsWith("fsub", Instruction::FSub)
1590 .StartsWith("fmuladd", 0)
1591 .StartsWith("fmul", Instruction::FMul)
1592 .StartsWith("fdiv", Instruction::FDiv)
1593 .StartsWith("frem", Instruction::FRem)
1594 .StartsWith("fneg", Instruction::FNeg)
1595 .StartsWith("trunc", Instruction::Trunc)
1596 .StartsWith("zext", Instruction::ZExt)
1597 .StartsWith("sext", Instruction::SExt)
1598 .StartsWith("fptrunc", Instruction::FPTrunc)
1599 .StartsWith("fpext", Instruction::FPExt)
1600 .StartsWith("fptoui", Instruction::FPToUI)
1601 .StartsWith("fptosi", Instruction::FPToSI)
1602 .StartsWith("uitofp", Instruction::UIToFP)
1603 .StartsWith("sitofp", Instruction::SIToFP)
1604 .StartsWith("ptrtoint", Instruction::PtrToInt)
1605 .StartsWith("inttoptr", Instruction::IntToPtr)
1606 .StartsWith("icmp", Instruction::ICmp)
1607 .StartsWith("fcmp", Instruction::FCmp)
1608 .Default(0);
1609}
1610
1611static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1612 if (!Name.consume_front("vp."))
1613 return 0;
1614 return StringSwitch<Intrinsic::ID>(Name)
1615 .StartsWith("abs", Intrinsic::abs)
1616 .StartsWith("smax", Intrinsic::smax)
1617 .StartsWith("smin", Intrinsic::smin)
1618 .StartsWith("umax", Intrinsic::umax)
1619 .StartsWith("umin", Intrinsic::umin)
1620 .StartsWith("copysign", Intrinsic::copysign)
1621 .StartsWith("minnum", Intrinsic::minnum)
1622 .StartsWith("maxnum", Intrinsic::maxnum)
1623 .StartsWith("minimum", Intrinsic::minimum)
1624 .StartsWith("maximum", Intrinsic::maximum)
1625 .StartsWith("fabs", Intrinsic::fabs)
1626 .StartsWith("sqrt", Intrinsic::sqrt)
1627 .StartsWith("fma", Intrinsic::fma)
1628 .StartsWith("fmuladd", Intrinsic::fmuladd)
1629 .StartsWith("ceil", Intrinsic::ceil)
1630 .StartsWith("floor", Intrinsic::floor)
1631 .StartsWith("rint", Intrinsic::rint)
1632 .StartsWith("nearbyint", Intrinsic::nearbyint)
1633 .StartsWith("roundeven", Intrinsic::roundeven)
1634 .StartsWith("roundtozero", Intrinsic::trunc)
1635 .StartsWith("round", Intrinsic::round)
1636 .StartsWith("lrint", Intrinsic::lrint)
1637 .StartsWith("llrint", Intrinsic::llrint)
1638 .StartsWith("bitreverse", Intrinsic::bitreverse)
1639 .StartsWith("bswap", Intrinsic::bswap)
1640 .StartsWith("ctpop", Intrinsic::ctpop)
1641 .StartsWith("ctlz", Intrinsic::ctlz)
1642 .StartsWith("cttz.elts", 0)
1643 .StartsWith("cttz", Intrinsic::cttz)
1644 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1645 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1646 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1647 .StartsWith("usub.sat", Intrinsic::usub_sat)
1648 .StartsWith("fshl", Intrinsic::fshl)
1649 .StartsWith("fshr", Intrinsic::fshr)
1650 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1651 .Default(0);
1652}
1653
1657
1658static bool convertIntrinsicValidType(StringRef Name,
1659 const FunctionType *FuncTy) {
1660 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1661 if (Name.starts_with("to.fp16")) {
1662 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1663 HalfTy) &&
1664 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1665 FuncTy->getReturnType());
1666 }
1667
1668 if (Name.starts_with("from.fp16")) {
1669 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1670 HalfTy) &&
1671 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1672 FuncTy->getReturnType());
1673 }
1674
1675 return false;
1676}
1677
1678static unsigned
1679getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1680 SmallVectorImpl<Type *> &OverloadTys) {
1681 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1682 if (Defaults.empty())
1683 return 0;
1684
1685 unsigned FullArgCount = FirstDefault + Defaults.size();
1686
1687 // Only trailing default arguments can be missing.
1688 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1689 return 0;
1690
1691 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1692 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1693 NumMissingTrailingParams))
1694 return 0;
1695
1696 return FullArgCount;
1697}
1698
1699static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1700 Intrinsic::ID IID = F->getIntrinsicID();
1701 SmallVector<Type *, 4> OverloadTys;
1702
1703 unsigned FullArgCount =
1704 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1705 if (FullArgCount == 0)
1706 return false;
1707
1708 rename(F);
1709 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1710 assert(NewFn->arg_size() == FullArgCount &&
1711 "total number of default args does not match intrinsic signature");
1712 return true;
1713}
1714
1715static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1716 bool CanUpgradeDebugIntrinsicsToRecords) {
1717 assert(F && "Illegal to upgrade a non-existent Function.");
1718
1719 StringRef Name = F->getName();
1720
1721 // Quickly eliminate it, if it's not a candidate.
1722 if (!Name.consume_front("llvm.") || Name.empty())
1723 return false;
1724
1725 switch (Name[0]) {
1726 default: break;
1727 case 'a': {
1728 bool IsArm = Name.consume_front("arm.");
1729 if (IsArm || Name.consume_front("aarch64.")) {
1730 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1731 return true;
1732 break;
1733 }
1734
1735 if (Name.consume_front("amdgcn.")) {
1736 if (Name == "alignbit") {
1737 // Target specific intrinsic became redundant
1738 NewFn = Intrinsic::getOrInsertDeclaration(
1739 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1740 return true;
1741 }
1742
1743 if (Name.consume_front("atomic.")) {
1744 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1745 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1746 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1747 // and usub_sat so there's no new declaration.
1748 NewFn = nullptr;
1749 return true;
1750 }
1751 break; // No other 'amdgcn.atomic.*'
1752 }
1753
1754 if (Name.starts_with("addrspacecast.nonnull")) {
1755 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1756 // so there's no new declaration.
1757 NewFn = nullptr;
1758 return true;
1759 }
1760
1761 switch (F->getIntrinsicID()) {
1762 default:
1763 break;
1764 // Legacy wmma iu intrinsics without the optional clamp operand.
1765 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1766 if (F->arg_size() == 7) {
1767 NewFn = nullptr;
1768 return true;
1769 }
1770 break;
1771 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1772 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1773 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1774 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1775 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1776 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1777 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1778 if (F->arg_size() == 8) {
1779 NewFn = nullptr;
1780 return true;
1781 }
1782 break;
1783 }
1784
1785 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1786 Name.consume_front("flat.atomic.")) {
1787 if (Name.starts_with("fadd") ||
1788 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1789 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1790 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1791 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1792 // declaration.
1793 NewFn = nullptr;
1794 return true;
1795 }
1796 }
1797
1798 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1799 NewFn = nullptr;
1800 return true;
1801 }
1802
1803 if (Name.starts_with("ldexp.")) {
1804 // Target specific intrinsic became redundant
1805 NewFn = Intrinsic::getOrInsertDeclaration(
1806 F->getParent(), Intrinsic::ldexp,
1807 {F->getReturnType(), F->getArg(1)->getType()});
1808 return true;
1809 }
1810 break; // No other 'amdgcn.*'
1811 }
1812
1813 break;
1814 }
1815 case 'c': {
1816 if (F->arg_size() == 1) {
1817 if (Name.consume_front("convert.")) {
1818 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1819 NewFn = nullptr;
1820 return true;
1821 }
1822 }
1823
1824 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1825 .StartsWith("ctlz.", Intrinsic::ctlz)
1826 .StartsWith("cttz.", Intrinsic::cttz)
1827 .Default(Intrinsic::not_intrinsic);
1828 if (ID != Intrinsic::not_intrinsic) {
1829 rename(F);
1830 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1831 F->arg_begin()->getType());
1832 return true;
1833 }
1834 }
1835
1836 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1837 if (Name == "coro.end" &&
1838 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1839 CoroEndID = Intrinsic::coro_end;
1840 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1841 CoroEndID = Intrinsic::coro_end_async;
1842
1843 if (CoroEndID != Intrinsic::not_intrinsic) {
1844 rename(F);
1845 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1846 return true;
1847 }
1848
1849 break;
1850 }
1851 case 'd':
1852 if (Name.consume_front("dbg.")) {
1853 // Mark debug intrinsics for upgrade to new debug format.
1854 if (CanUpgradeDebugIntrinsicsToRecords) {
1855 if (Name == "addr" || Name == "value" || Name == "assign" ||
1856 Name == "declare" || Name == "label") {
1857 // There's no function to replace these with.
1858 NewFn = nullptr;
1859 // But we do want these to get upgraded.
1860 return true;
1861 }
1862 }
1863 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1864 // converted to DbgVariableRecords later.
1865 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1866 rename(F);
1867 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1868 Intrinsic::dbg_value);
1869 return true;
1870 }
1871 break; // No other 'dbg.*'.
1872 }
1873 break;
1874 case 'e':
1875 if (Name.consume_front("experimental.vector.")) {
1876 Intrinsic::ID ID =
1877 StringSwitch<Intrinsic::ID>(Name)
1878 // Skip over extract.last.active, otherwise it will be 'upgraded'
1879 // to a regular vector extract which is a different operation.
1880 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1881 .StartsWith("extract.", Intrinsic::vector_extract)
1882 .StartsWith("insert.", Intrinsic::vector_insert)
1883 .StartsWith("reverse.", Intrinsic::vector_reverse)
1884 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1885 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1886 .StartsWith("partial.reduce.add",
1887 Intrinsic::vector_partial_reduce_add)
1888 .Default(Intrinsic::not_intrinsic);
1889 if (ID != Intrinsic::not_intrinsic) {
1890 const auto *FT = F->getFunctionType();
1891 SmallVector<Type *, 2> Tys;
1892 if (ID == Intrinsic::vector_extract ||
1893 ID == Intrinsic::vector_interleave2)
1894 // Extracting overloads the return type.
1895 Tys.push_back(FT->getReturnType());
1896 if (ID != Intrinsic::vector_interleave2)
1897 Tys.push_back(FT->getParamType(0));
1898 if (ID == Intrinsic::vector_insert ||
1899 ID == Intrinsic::vector_partial_reduce_add)
1900 // Inserting overloads the inserted type.
1901 Tys.push_back(FT->getParamType(1));
1902 rename(F);
1903 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1904 return true;
1905 }
1906
1907 if (Name.consume_front("reduce.")) {
1908 SmallVector<StringRef, 2> Groups;
1909 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1910 if (R.match(Name, &Groups))
1911 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1912 .Case("add", Intrinsic::vector_reduce_add)
1913 .Case("mul", Intrinsic::vector_reduce_mul)
1914 .Case("and", Intrinsic::vector_reduce_and)
1915 .Case("or", Intrinsic::vector_reduce_or)
1916 .Case("xor", Intrinsic::vector_reduce_xor)
1917 .Case("smax", Intrinsic::vector_reduce_smax)
1918 .Case("smin", Intrinsic::vector_reduce_smin)
1919 .Case("umax", Intrinsic::vector_reduce_umax)
1920 .Case("umin", Intrinsic::vector_reduce_umin)
1921 .Case("fmax", Intrinsic::vector_reduce_fmax)
1922 .Case("fmin", Intrinsic::vector_reduce_fmin)
1923 .Default(Intrinsic::not_intrinsic);
1924
1925 bool V2 = false;
1926 if (ID == Intrinsic::not_intrinsic) {
1927 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1928 Groups.clear();
1929 V2 = true;
1930 if (R2.match(Name, &Groups))
1931 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1932 .Case("fadd", Intrinsic::vector_reduce_fadd)
1933 .Case("fmul", Intrinsic::vector_reduce_fmul)
1934 .Default(Intrinsic::not_intrinsic);
1935 }
1936 if (ID != Intrinsic::not_intrinsic) {
1937 rename(F);
1938 auto Args = F->getFunctionType()->params();
1939 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1940 {Args[V2 ? 1 : 0]});
1941 return true;
1942 }
1943 break; // No other 'expermental.vector.reduce.*'.
1944 }
1945
1946 if (Name.consume_front("splice"))
1947 return true;
1948 break; // No other 'experimental.vector.*'.
1949 }
1950 if (Name.consume_front("experimental.stepvector.")) {
1951 Intrinsic::ID ID = Intrinsic::stepvector;
1952 rename(F);
1953 NewFn = Intrinsic::getOrInsertDeclaration(
1954 F->getParent(), ID, F->getFunctionType()->getReturnType());
1955 return true;
1956 }
1957 break; // No other 'e*'.
1958 case 'f':
1959 if (Name.starts_with("flt.rounds")) {
1960 rename(F);
1961 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1962 Intrinsic::get_rounding);
1963 return true;
1964 }
1965 break;
1966 case 'i':
1967 if (Name.starts_with("invariant.group.barrier")) {
1968 // Rename invariant.group.barrier to launder.invariant.group
1969 auto Args = F->getFunctionType()->params();
1970 Type* ObjectPtr[1] = {Args[0]};
1971 rename(F);
1972 NewFn = Intrinsic::getOrInsertDeclaration(
1973 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1974 return true;
1975 }
1976 break;
1977 case 'l': {
1978 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1979 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1980 if (IsLifetimeStart || IsLifetimeEnd) {
1981 if (F->arg_size() == 2) {
1982 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1983 : Intrinsic::lifetime_end;
1984 rename(F);
1985 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1986 // arguments. Use the Ptr argument to create new declaration.
1987 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1988 F->getArg(1)->getType());
1989 return true;
1990 } else if (F->arg_size() == 1 && Name == ".i64") {
1991 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1992 // Autoupgrade prior to
1993 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1994 // intrinsic with no expected calls. To allow auto-upgrade process to
1995 // delete such invalid intrinsic declaration, set NewFn = nullptr
1996 // and return true here. If there are actual calls to this intrinsic
1997 // (which is not expected), they will be deleted in
1998 // UpgradeIntrinsicCall.
1999 NewFn = nullptr;
2000 return true;
2001 }
2002 }
2003 break;
2004 }
2005 case 'm': {
2006 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2007 // alignment parameter to embedding the alignment as an attribute of
2008 // the pointer args.
2009 if (unsigned ID = StringSwitch<unsigned>(Name)
2010 .StartsWith("memcpy.", Intrinsic::memcpy)
2011 .StartsWith("memmove.", Intrinsic::memmove)
2012 .Default(0)) {
2013 if (F->arg_size() == 5) {
2014 rename(F);
2015 // Get the types of dest, src, and len
2016 ArrayRef<Type *> ParamTypes =
2017 F->getFunctionType()->params().slice(0, 3);
2018 NewFn =
2019 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2020 return true;
2021 }
2022 }
2023 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2024 rename(F);
2025 // Get the types of dest, and len
2026 const auto *FT = F->getFunctionType();
2027 Type *ParamTypes[2] = {
2028 FT->getParamType(0), // Dest
2029 FT->getParamType(2) // len
2030 };
2031 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2032 Intrinsic::memset, ParamTypes);
2033 return true;
2034 }
2035
2036 unsigned MaskedID =
2037 StringSwitch<unsigned>(Name)
2038 .StartsWith("masked.load", Intrinsic::masked_load)
2039 .StartsWith("masked.gather", Intrinsic::masked_gather)
2040 .StartsWith("masked.store", Intrinsic::masked_store)
2041 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2042 .Default(0);
2043 if (MaskedID && F->arg_size() == 4) {
2044 rename(F);
2045 if (MaskedID == Intrinsic::masked_load ||
2046 MaskedID == Intrinsic::masked_gather) {
2047 NewFn = Intrinsic::getOrInsertDeclaration(
2048 F->getParent(), MaskedID,
2049 {F->getReturnType(), F->getArg(0)->getType()});
2050 return true;
2051 }
2052 NewFn = Intrinsic::getOrInsertDeclaration(
2053 F->getParent(), MaskedID,
2054 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2055 return true;
2056 }
2057 break;
2058 }
2059 case 'n': {
2060 if (Name.consume_front("nvvm.")) {
2061 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2062 if (F->arg_size() == 1) {
2063 Intrinsic::ID IID =
2064 StringSwitch<Intrinsic::ID>(Name)
2065 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2066 .Case("clz.i", Intrinsic::ctlz)
2067 .Case("popc.i", Intrinsic::ctpop)
2068 .Default(Intrinsic::not_intrinsic);
2069 if (IID != Intrinsic::not_intrinsic) {
2070 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2071 {F->getReturnType()});
2072 return true;
2073 }
2074 } else if (F->arg_size() == 2) {
2075 Intrinsic::ID IID =
2076 StringSwitch<Intrinsic::ID>(Name)
2077 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2078 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2079 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2080 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2081 .Cases({"mulhi.s", "mulhi.i", "mulhi.ll"}, Intrinsic::smulh)
2082 .Cases({"mulhi.us", "mulhi.ui", "mulhi.ull"}, Intrinsic::umulh)
2083 .Default(Intrinsic::not_intrinsic);
2084 if (IID != Intrinsic::not_intrinsic) {
2085 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2086 {F->getReturnType()});
2087 return true;
2088 }
2089 }
2090
2091 // Check for nvvm intrinsics that need a return type adjustment.
2092 {
2093 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2094 if (IID != Intrinsic::not_intrinsic &&
2095 isLegacyNVPTXBF16IntSignature(F, IID)) {
2096 NewFn = nullptr;
2097 return true;
2098 }
2099 }
2100
2101 // Upgrade Distributed Shared Memory Intrinsics
2102 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2103 if (IID != Intrinsic::not_intrinsic) {
2104 rename(F);
2105 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2106 return true;
2107 }
2108
2109 // Upgrade TMA reduction intrinsics
2110 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2111 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2112 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2113 if (IID != Intrinsic::not_intrinsic) {
2114 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2115 return true;
2116 }
2117
2118 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2119 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2120 if (IID != Intrinsic::not_intrinsic) {
2121 rename(F);
2122 NewFn = Intrinsic::getOrInsertDeclaration(
2123 F->getParent(), IID, F->getReturnType(),
2124 F->getFunctionType()->params());
2125 return true;
2126 }
2127
2128 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2129 // tcgen05.alloc shared variants to anyptr intrinsics.
2130 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2131 if (IID != Intrinsic::not_intrinsic) {
2132 rename(F);
2133 if (Intrinsic::isOverloaded(IID))
2134 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2135 {F->getArg(0)->getType()});
2136 else
2137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2138 return true;
2139 }
2140
2141 // Upgrade TMA copy G2S CTA intrinsics.
2142 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2143 if (IID != Intrinsic::not_intrinsic) {
2144 rename(F);
2145 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2146 return true;
2147 }
2148
2149 // Upgrade TMA copy G2S (cluster) intrinsics.
2150 SmallVector<Type *, 1> OvlTys;
2151 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2152 if (IID != Intrinsic::not_intrinsic) {
2153 rename(F);
2154 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2155 return true;
2156 }
2157
2158 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2159 // (multicast-mask overloading + trailing flag_valid_pattern).
2160 SmallVector<Type *, 1> BulkG2SOvlTys;
2161 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2162 if (IID != Intrinsic::not_intrinsic) {
2163 rename(F);
2164 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2165 BulkG2SOvlTys);
2166 return true;
2167 }
2168
2169 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2170 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2171 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2172 if (IID != Intrinsic::not_intrinsic) {
2173 rename(F);
2174 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2175 return true;
2176 }
2177
2178 // Upgrade mbarrier.init intrinsics missing the layout operand.
2179 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2180 if (IID != Intrinsic::not_intrinsic) {
2181 rename(F);
2182 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2183 F->getArg(0)->getType());
2184 return true;
2185 }
2186
2187 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2188 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2189 //
2190 // TODO: We could add lohi.i2d.
2191 bool Expand = false;
2192 if (Name.consume_front("abs."))
2193 // nvvm.abs.{i,ii}
2194 Expand =
2195 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2196 else if (Name.consume_front("fabs."))
2197 // nvvm.fabs.{f,ftz.f,d}
2198 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2199 else if (Name.consume_front("add."))
2200 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2201 Expand = getNVVMFPArithUpgrade(Name, NVVMFAddIIDs).has_value();
2202 else if (Name.consume_front("mul."))
2203 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2204 Expand = getNVVMFPArithUpgrade(Name, NVVMFMulIIDs).has_value();
2205 else if (Name.consume_front("ex2.approx."))
2206 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2207 Expand =
2208 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2209 else if (Name.consume_front("atomic.load."))
2210 // nvvm.atomic.load.add.{f32,f64}.p
2211 // nvvm.atomic.load.{inc,dec}.32.p
2212 Expand = StringSwitch<bool>(Name)
2213 .StartsWith("add.f32.p", true)
2214 .StartsWith("add.f64.p", true)
2215 .StartsWith("inc.32.p", true)
2216 .StartsWith("dec.32.p", true)
2217 .Default(false);
2218 else if (Name.consume_front("atomic."))
2219 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2220 // nvvm.atomic.cas.gen.i.{cta,sys}
2221 Expand = StringSwitch<bool>(Name)
2222 .StartsWith("add.gen.", true)
2223 .StartsWith("exch.gen.", true)
2224 .StartsWith("max.gen.", true)
2225 .StartsWith("min.gen.", true)
2226 .StartsWith("inc.gen.", true)
2227 .StartsWith("dec.gen.", true)
2228 .StartsWith("and.gen.", true)
2229 .StartsWith("or.gen.", true)
2230 .StartsWith("xor.gen.", true)
2231 .StartsWith("cas.gen.", true)
2232 .Default(false);
2233 else if (Name.consume_front("bitcast."))
2234 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2235 Expand =
2236 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2237 else if (Name.consume_front("rotate."))
2238 // nvvm.rotate.{b32,b64,right.b64}
2239 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2240 else if (Name.consume_front("ptr.gen.to."))
2241 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2242 Expand = consumeNVVMPtrAddrSpace(Name);
2243 else if (Name.consume_front("ptr."))
2244 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2245 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2246 else if (Name.consume_front("ldg.global."))
2247 // nvvm.ldg.global.{i,p,f}
2248 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2249 Name.starts_with("p."));
2250 else
2251 Expand = StringSwitch<bool>(Name)
2252 .Case("barrier0", true)
2253 .Case("barrier.n", true)
2254 .Case("barrier.sync.cnt", true)
2255 .Case("barrier.sync", true)
2256 .Case("barrier", true)
2257 .Case("bar.sync", true)
2258 .Case("barrier0.popc", true)
2259 .Case("barrier0.and", true)
2260 .Case("barrier0.or", true)
2261 .Case("clz.ll", true)
2262 .Case("popc.ll", true)
2263 .Case("h2f", true)
2264 .Case("swap.lo.hi.b64", true)
2265 .Case("tanh.approx.f32", true)
2266 .Default(false);
2267
2268 if (Expand) {
2269 NewFn = nullptr;
2270 return true;
2271 }
2272 break; // No other 'nvvm.*'.
2273 }
2274 break;
2275 }
2276 case 'o':
2277 if (Name.starts_with("objectsize.")) {
2278 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2279 if (F->arg_size() == 2 || F->arg_size() == 3) {
2280 rename(F);
2281 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2282 Intrinsic::objectsize, Tys);
2283 return true;
2284 }
2285 }
2286 break;
2287
2288 case 'p':
2289 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2290 rename(F);
2291 NewFn = Intrinsic::getOrInsertDeclaration(
2292 F->getParent(), Intrinsic::ptr_annotation,
2293 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2294 return true;
2295 }
2296 break;
2297
2298 case 'r': {
2299 if (Name.consume_front("riscv.")) {
2300 Intrinsic::ID ID;
2301 ID = StringSwitch<Intrinsic::ID>(Name)
2302 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2303 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2304 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2305 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2306 .Default(Intrinsic::not_intrinsic);
2307 if (ID != Intrinsic::not_intrinsic) {
2308 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2309 rename(F);
2310 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2311 return true;
2312 }
2313 break; // No other applicable upgrades.
2314 }
2315
2316 ID = StringSwitch<Intrinsic::ID>(Name)
2317 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2318 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2319 .Default(Intrinsic::not_intrinsic);
2320 if (ID != Intrinsic::not_intrinsic) {
2321 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2322 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2323 rename(F);
2324 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2325 return true;
2326 }
2327 break; // No other applicable upgrades.
2328 }
2329
2330 ID = StringSwitch<Intrinsic::ID>(Name)
2331 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2332 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2333 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2334 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2335 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2336 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2337 .Default(Intrinsic::not_intrinsic);
2338 if (ID != Intrinsic::not_intrinsic) {
2339 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2340 rename(F);
2341 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2342 return true;
2343 }
2344 break; // No other applicable upgrades.
2345 }
2346
2347 // Replace llvm.riscv.clmul with llvm.clmul.
2348 if (Name == "clmul.i32" || Name == "clmul.i64") {
2349 NewFn = Intrinsic::getOrInsertDeclaration(
2350 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2351 return true;
2352 }
2353
2354 break; // No other 'riscv.*' intrinsics
2355 }
2356 } break;
2357
2358 case 's':
2359 if (Name == "stackprotectorcheck") {
2360 NewFn = nullptr;
2361 return true;
2362 }
2363 if (Name.starts_with("strip.invariant.group")) {
2364 // For clang's usage it would be safe to just drop the
2365 // strip.invariant.group, but to be conservative replace with the
2366 // stronger launder.invariant.group instead.
2367 NewFn = Intrinsic::getOrInsertDeclaration(
2368 F->getParent(), Intrinsic::launder_invariant_group,
2369 F->getReturnType());
2370 return true;
2371 }
2372 break;
2373
2374 case 't':
2375 if (Name == "thread.pointer") {
2376 NewFn = Intrinsic::getOrInsertDeclaration(
2377 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2378 return true;
2379 }
2380 break;
2381
2382 case 'v': {
2383 if (Name == "var.annotation" && F->arg_size() == 4) {
2384 rename(F);
2385 NewFn = Intrinsic::getOrInsertDeclaration(
2386 F->getParent(), Intrinsic::var_annotation,
2387 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2388 return true;
2389 }
2390 if (Name.consume_front("vector.splice")) {
2391 if (Name.starts_with(".left") || Name.starts_with(".right"))
2392 break;
2393 return true;
2394 }
2395 if (shouldUpgradeVPIntrinsic(Name))
2396 return true;
2397 break;
2398 }
2399
2400 case 'w':
2401 if (Name.consume_front("wasm.")) {
2402 Intrinsic::ID ID =
2403 StringSwitch<Intrinsic::ID>(Name)
2404 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2405 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2406 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2407 .Default(Intrinsic::not_intrinsic);
2408 if (ID != Intrinsic::not_intrinsic) {
2409 rename(F);
2410 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2411 F->getReturnType());
2412 return true;
2413 }
2414
2415 if (Name.consume_front("dot.i8x16.i7x16.")) {
2416 ID = StringSwitch<Intrinsic::ID>(Name)
2417 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2418 .Case("add.signed",
2419 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2420 .Default(Intrinsic::not_intrinsic);
2421 if (ID != Intrinsic::not_intrinsic) {
2422 rename(F);
2423 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2424 return true;
2425 }
2426 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2427 }
2428 break; // No other 'wasm.*'.
2429 }
2430 break;
2431
2432 case 'x':
2433 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2434 return true;
2435 }
2436
2437 auto *ST = dyn_cast<StructType>(F->getReturnType());
2438 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2439 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2440 // Replace return type with literal non-packed struct. Only do this for
2441 // intrinsics declared to return a struct, not for intrinsics with
2442 // overloaded return type, in which case the exact struct type will be
2443 // mangled into the name.
2444 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2445 FunctionType *FT = F->getFunctionType();
2446 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2447 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2448 std::string Name = F->getName().str();
2449 rename(F);
2450 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2451 Name, F->getParent());
2452
2453 // The new function may also need remangling.
2454 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2455 NewFn = *Result;
2456 return true;
2457 }
2458 }
2459
2460 // Remangle our intrinsic since we upgrade the mangling
2461 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2462 if (Result != std::nullopt) {
2463 NewFn = *Result;
2464 return true;
2465 }
2466
2467 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2468 return true;
2469
2470 // This may not belong here. This function is effectively being overloaded
2471 // to both detect an intrinsic which needs upgrading, and to provide the
2472 // upgraded form of the intrinsic. We should perhaps have two separate
2473 // functions for this.
2474
2475 return false;
2476}
2477
2478bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2479 bool CanUpgradeDebugIntrinsicsToRecords) {
2480 NewFn = nullptr;
2481 bool Upgraded =
2482 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2483
2484 // Upgrade intrinsic attributes. This does not change the function.
2485 if (NewFn)
2486 F = NewFn;
2487 if (Intrinsic::ID id = F->getIntrinsicID()) {
2488 // Only do this if the intrinsic signature is valid.
2489 SmallVector<Type *> OverloadTys;
2490 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2491 F->setAttributes(
2492 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2493 }
2494 return Upgraded;
2495}
2496
2497GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2498 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2499 GV->getName() == "llvm.global_dtors")) ||
2500 !GV->hasInitializer())
2501 return nullptr;
2502 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2503 if (!ATy)
2504 return nullptr;
2505 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2506 if (!STy || STy->getNumElements() != 2)
2507 return nullptr;
2508
2509 IRBuilder<> IRB(*GV->getParent());
2510 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2511 IRB.getPtrTy());
2512 Constant *Init = GV->getInitializer();
2513 unsigned N = Init->getNumOperands();
2514 std::vector<Constant *> NewCtors(N);
2515 for (unsigned i = 0; i != N; ++i) {
2516 auto Ctor = cast<Constant>(Init->getOperand(i));
2517 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2518 Ctor->getAggregateElement(1),
2519 ConstantPointerNull::get(IRB.getPtrTy()));
2520 }
2521 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2522
2523 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2524 NewInit, GV->getName());
2525}
2526
2527// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2528// to byte shuffles.
2529static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2530 unsigned Shift) {
2531 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2532 unsigned NumElts = ResultTy->getNumElements() * 8;
2533
2534 // Bitcast from a 64-bit element type to a byte element type.
2535 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2536 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2537
2538 // We'll be shuffling in zeroes.
2539 Value *Res = Constant::getNullValue(VecTy);
2540
2541 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2542 // we'll just return the zero vector.
2543 if (Shift < 16) {
2544 int Idxs[64];
2545 // 256/512-bit version is split into 2/4 16-byte lanes.
2546 for (unsigned l = 0; l != NumElts; l += 16)
2547 for (unsigned i = 0; i != 16; ++i) {
2548 unsigned Idx = NumElts + i - Shift;
2549 if (Idx < NumElts)
2550 Idx -= NumElts - 16; // end of lane, switch operand.
2551 Idxs[l + i] = Idx + l;
2552 }
2553
2554 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2555 }
2556
2557 // Bitcast back to a 64-bit element type.
2558 return Builder.CreateBitCast(Res, ResultTy, "cast");
2559}
2560
2561// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2562// to byte shuffles.
2563static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2564 unsigned Shift) {
2565 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2566 unsigned NumElts = ResultTy->getNumElements() * 8;
2567
2568 // Bitcast from a 64-bit element type to a byte element type.
2569 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2570 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2571
2572 // We'll be shuffling in zeroes.
2573 Value *Res = Constant::getNullValue(VecTy);
2574
2575 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2576 // we'll just return the zero vector.
2577 if (Shift < 16) {
2578 int Idxs[64];
2579 // 256/512-bit version is split into 2/4 16-byte lanes.
2580 for (unsigned l = 0; l != NumElts; l += 16)
2581 for (unsigned i = 0; i != 16; ++i) {
2582 unsigned Idx = i + Shift;
2583 if (Idx >= 16)
2584 Idx += NumElts - 16; // end of lane, switch operand.
2585 Idxs[l + i] = Idx + l;
2586 }
2587
2588 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2589 }
2590
2591 // Bitcast back to a 64-bit element type.
2592 return Builder.CreateBitCast(Res, ResultTy, "cast");
2593}
2594
2595static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2596 unsigned NumElts) {
2597 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2598 llvm::VectorType *MaskTy = FixedVectorType::get(
2599 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2600 Mask = Builder.CreateBitCast(Mask, MaskTy);
2601
2602 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2603 // i8 and we need to extract down to the right number of elements.
2604 if (NumElts <= 4) {
2605 int Indices[4];
2606 for (unsigned i = 0; i != NumElts; ++i)
2607 Indices[i] = i;
2608 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2609 "extract");
2610 }
2611
2612 return Mask;
2613}
2614
2615static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2616 Value *Op1) {
2617 // If the mask is all ones just emit the first operation.
2618 if (const auto *C = dyn_cast<Constant>(Mask))
2619 if (C->isAllOnesValue())
2620 return Op0;
2621
2622 Mask = getX86MaskVec(Builder, Mask,
2623 cast<FixedVectorType>(Op0->getType())->getNumElements());
2624 return Builder.CreateSelect(Mask, Op0, Op1);
2625}
2626
2627static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2628 Value *Op1) {
2629 // If the mask is all ones just emit the first operation.
2630 if (const auto *C = dyn_cast<Constant>(Mask))
2631 if (C->isAllOnesValue())
2632 return Op0;
2633
2634 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2635 Mask->getType()->getIntegerBitWidth());
2636 Mask = Builder.CreateBitCast(Mask, MaskTy);
2637 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2638 return Builder.CreateSelect(Mask, Op0, Op1);
2639}
2640
2641// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2642// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2643// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2644static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2645 Value *Op1, Value *Shift,
2646 Value *Passthru, Value *Mask,
2647 bool IsVALIGN) {
2648 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2649
2650 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2651 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2652 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2653 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2654
2655 // Mask the immediate for VALIGN.
2656 if (IsVALIGN)
2657 ShiftVal &= (NumElts - 1);
2658
2659 // If palignr is shifting the pair of vectors more than the size of two
2660 // lanes, emit zero.
2661 if (ShiftVal >= 32)
2662 return llvm::Constant::getNullValue(Op0->getType());
2663
2664 // If palignr is shifting the pair of input vectors more than one lane,
2665 // but less than two lanes, convert to shifting in zeroes.
2666 if (ShiftVal > 16) {
2667 ShiftVal -= 16;
2668 Op1 = Op0;
2669 Op0 = llvm::Constant::getNullValue(Op0->getType());
2670 }
2671
2672 int Indices[64];
2673 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2674 for (unsigned l = 0; l < NumElts; l += 16) {
2675 for (unsigned i = 0; i != 16; ++i) {
2676 unsigned Idx = ShiftVal + i;
2677 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2678 Idx += NumElts - 16; // End of lane, switch operand.
2679 Indices[l + i] = Idx + l;
2680 }
2681 }
2682
2683 Value *Align = Builder.CreateShuffleVector(
2684 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2685
2686 return emitX86Select(Builder, Mask, Align, Passthru);
2687}
2688
2689static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2690 bool ZeroMask, bool IndexForm) {
2691 Type *Ty = CI.getType();
2692 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2693 unsigned EltWidth = Ty->getScalarSizeInBits();
2694 bool IsFloat = Ty->isFPOrFPVectorTy();
2695 Intrinsic::ID IID;
2696 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2697 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2698 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2699 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2700 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2701 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2702 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2703 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2704 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2705 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2706 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2707 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2708 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2709 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2710 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2711 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2712 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2713 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2714 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2715 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2716 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2717 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2718 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2719 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2720 else if (VecWidth == 128 && EltWidth == 16)
2721 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2722 else if (VecWidth == 256 && EltWidth == 16)
2723 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2724 else if (VecWidth == 512 && EltWidth == 16)
2725 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2726 else if (VecWidth == 128 && EltWidth == 8)
2727 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2728 else if (VecWidth == 256 && EltWidth == 8)
2729 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2730 else if (VecWidth == 512 && EltWidth == 8)
2731 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2732 else
2733 llvm_unreachable("Unexpected intrinsic");
2734
2735 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2736 CI.getArgOperand(2) };
2737
2738 // If this isn't index form we need to swap operand 0 and 1.
2739 if (!IndexForm)
2740 std::swap(Args[0], Args[1]);
2741
2742 Value *V = Builder.CreateIntrinsic(IID, Args);
2743 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2744 : Builder.CreateBitCast(CI.getArgOperand(1),
2745 Ty);
2746 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2747}
2748
2749static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2750 Intrinsic::ID IID) {
2751 Type *Ty = CI.getType();
2752 Value *Op0 = CI.getOperand(0);
2753 Value *Op1 = CI.getOperand(1);
2754 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2755
2756 if (CI.arg_size() == 4) { // For masked intrinsics.
2757 Value *VecSrc = CI.getOperand(2);
2758 Value *Mask = CI.getOperand(3);
2759 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2760 }
2761 return Res;
2762}
2763
2764static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2765 bool IsRotateRight) {
2766 Type *Ty = CI.getType();
2767 Value *Src = CI.getArgOperand(0);
2768 Value *Amt = CI.getArgOperand(1);
2769
2770 // Amount may be scalar immediate, in which case create a splat vector.
2771 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2772 // we only care about the lowest log2 bits anyway.
2773 if (Amt->getType() != Ty) {
2774 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2775 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2776 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2777 }
2778
2779 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2780 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2781
2782 if (CI.arg_size() == 4) { // For masked intrinsics.
2783 Value *VecSrc = CI.getOperand(2);
2784 Value *Mask = CI.getOperand(3);
2785 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2786 }
2787 return Res;
2788}
2789
2790static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2791 bool IsSigned) {
2792 Type *Ty = CI.getType();
2793 Value *LHS = CI.getArgOperand(0);
2794 Value *RHS = CI.getArgOperand(1);
2795
2796 CmpInst::Predicate Pred;
2797 switch (Imm) {
2798 case 0x0:
2799 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2800 break;
2801 case 0x1:
2802 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2803 break;
2804 case 0x2:
2805 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2806 break;
2807 case 0x3:
2808 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2809 break;
2810 case 0x4:
2811 Pred = ICmpInst::ICMP_EQ;
2812 break;
2813 case 0x5:
2814 Pred = ICmpInst::ICMP_NE;
2815 break;
2816 case 0x6:
2817 return Constant::getNullValue(Ty); // FALSE
2818 case 0x7:
2819 return Constant::getAllOnesValue(Ty); // TRUE
2820 default:
2821 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2822 }
2823
2824 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2825 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2826 return Ext;
2827}
2828
2829static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2830 bool IsShiftRight, bool ZeroMask) {
2831 Type *Ty = CI.getType();
2832 Value *Op0 = CI.getArgOperand(0);
2833 Value *Op1 = CI.getArgOperand(1);
2834 Value *Amt = CI.getArgOperand(2);
2835
2836 if (IsShiftRight)
2837 std::swap(Op0, Op1);
2838
2839 // Amount may be scalar immediate, in which case create a splat vector.
2840 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2841 // we only care about the lowest log2 bits anyway.
2842 if (Amt->getType() != Ty) {
2843 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2844 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2845 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2846 }
2847
2848 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2849 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2850
2851 unsigned NumArgs = CI.arg_size();
2852 if (NumArgs >= 4) { // For masked intrinsics.
2853 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2854 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2855 CI.getArgOperand(0);
2856 Value *Mask = CI.getOperand(NumArgs - 1);
2857 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2858 }
2859 return Res;
2860}
2861
2862static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2863 Value *Mask, bool Aligned) {
2864 const Align Alignment =
2865 Aligned
2866 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2867 : Align(1);
2868
2869 // If the mask is all ones just emit a regular store.
2870 if (const auto *C = dyn_cast<Constant>(Mask))
2871 if (C->isAllOnesValue())
2872 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2873
2874 // Convert the mask from an integer type to a vector of i1.
2875 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2876 Mask = getX86MaskVec(Builder, Mask, NumElts);
2877 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2878}
2879
2880static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2881 Value *Passthru, Value *Mask, bool Aligned) {
2882 Type *ValTy = Passthru->getType();
2883 const Align Alignment =
2884 Aligned
2885 ? Align(
2886 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2887 8)
2888 : Align(1);
2889
2890 // If the mask is all ones just emit a regular store.
2891 if (const auto *C = dyn_cast<Constant>(Mask))
2892 if (C->isAllOnesValue())
2893 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2894
2895 // Convert the mask from an integer type to a vector of i1.
2896 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2897 Mask = getX86MaskVec(Builder, Mask, NumElts);
2898 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2899}
2900
2901static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2902 Type *Ty = CI.getType();
2903 Value *Op0 = CI.getArgOperand(0);
2904 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2905 {Op0, Builder.getInt1(false)});
2906 if (CI.arg_size() == 3)
2907 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2908 return Res;
2909}
2910
2911static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2912 Type *Ty = CI.getType();
2913
2914 // Arguments have a vXi32 type so cast to vXi64.
2915 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2916 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2917
2918 if (IsSigned) {
2919 // Shift left then arithmetic shift right.
2920 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2921 LHS = Builder.CreateShl(LHS, ShiftAmt);
2922 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2923 RHS = Builder.CreateShl(RHS, ShiftAmt);
2924 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2925 } else {
2926 // Clear the upper bits.
2927 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2928 LHS = Builder.CreateAnd(LHS, Mask);
2929 RHS = Builder.CreateAnd(RHS, Mask);
2930 }
2931
2932 Value *Res = Builder.CreateMul(LHS, RHS);
2933
2934 if (CI.arg_size() == 4)
2935 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2936
2937 return Res;
2938}
2939
2940// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2941static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2942 Value *Mask) {
2943 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2944 if (Mask) {
2945 const auto *C = dyn_cast<Constant>(Mask);
2946 if (!C || !C->isAllOnesValue())
2947 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2948 }
2949
2950 if (NumElts < 8) {
2951 int Indices[8];
2952 for (unsigned i = 0; i != NumElts; ++i)
2953 Indices[i] = i;
2954 for (unsigned i = NumElts; i != 8; ++i)
2955 Indices[i] = NumElts + i % NumElts;
2956 Vec = Builder.CreateShuffleVector(Vec,
2957 Constant::getNullValue(Vec->getType()),
2958 Indices);
2959 }
2960 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2961}
2962
2963static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2964 unsigned CC, bool Signed) {
2965 Value *Op0 = CI.getArgOperand(0);
2966 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2967
2968 Value *Cmp;
2969 if (CC == 3) {
2970 Cmp = Constant::getNullValue(
2971 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2972 } else if (CC == 7) {
2973 Cmp = Constant::getAllOnesValue(
2974 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2975 } else {
2976 ICmpInst::Predicate Pred;
2977 switch (CC) {
2978 default: llvm_unreachable("Unknown condition code");
2979 case 0: Pred = ICmpInst::ICMP_EQ; break;
2980 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2981 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2982 case 4: Pred = ICmpInst::ICMP_NE; break;
2983 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2984 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2985 }
2986 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2987 }
2988
2989 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2990
2991 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2992}
2993
2994// Replace a masked intrinsic with an older unmasked intrinsic.
2995static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2996 Intrinsic::ID IID) {
2997 Value *Rep =
2998 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2999 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
3000}
3001
3002static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3003 Value* A = CI.getArgOperand(0);
3004 Value* B = CI.getArgOperand(1);
3005 Value* Src = CI.getArgOperand(2);
3006 Value* Mask = CI.getArgOperand(3);
3007
3008 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
3009 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3010 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
3011 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
3012 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3013 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3014}
3015
3016static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3017 Value* Op = CI.getArgOperand(0);
3018 Type* ReturnOp = CI.getType();
3019 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3020 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3021 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3022}
3023
3024// Replace intrinsic with unmasked version and a select.
3025static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3026 CallBase &CI, Value *&Rep) {
3027 Name = Name.substr(12); // Remove avx512.mask.
3028
3029 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3030 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3031 Intrinsic::ID IID;
3032 if (Name.starts_with("max.p")) {
3033 if (VecWidth == 128 && EltWidth == 32)
3034 IID = Intrinsic::x86_sse_max_ps;
3035 else if (VecWidth == 128 && EltWidth == 64)
3036 IID = Intrinsic::x86_sse2_max_pd;
3037 else if (VecWidth == 256 && EltWidth == 32)
3038 IID = Intrinsic::x86_avx_max_ps_256;
3039 else if (VecWidth == 256 && EltWidth == 64)
3040 IID = Intrinsic::x86_avx_max_pd_256;
3041 else
3042 llvm_unreachable("Unexpected intrinsic");
3043 } else if (Name.starts_with("min.p")) {
3044 if (VecWidth == 128 && EltWidth == 32)
3045 IID = Intrinsic::x86_sse_min_ps;
3046 else if (VecWidth == 128 && EltWidth == 64)
3047 IID = Intrinsic::x86_sse2_min_pd;
3048 else if (VecWidth == 256 && EltWidth == 32)
3049 IID = Intrinsic::x86_avx_min_ps_256;
3050 else if (VecWidth == 256 && EltWidth == 64)
3051 IID = Intrinsic::x86_avx_min_pd_256;
3052 else
3053 llvm_unreachable("Unexpected intrinsic");
3054 } else if (Name.starts_with("pshuf.b.")) {
3055 if (VecWidth == 128)
3056 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3057 else if (VecWidth == 256)
3058 IID = Intrinsic::x86_avx2_pshuf_b;
3059 else if (VecWidth == 512)
3060 IID = Intrinsic::x86_avx512_pshuf_b_512;
3061 else
3062 llvm_unreachable("Unexpected intrinsic");
3063 } else if (Name.starts_with("pmul.hr.sw.")) {
3064 if (VecWidth == 128)
3065 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3066 else if (VecWidth == 256)
3067 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3068 else if (VecWidth == 512)
3069 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3070 else
3071 llvm_unreachable("Unexpected intrinsic");
3072 } else if (Name.starts_with("pmulh.w")) {
3073 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3074 "Unexpected intrinsic");
3075 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::smulh);
3076 return true;
3077 } else if (Name.starts_with("pmulhu.w")) {
3078 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3079 "Unexpected intrinsic");
3080 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::umulh);
3081 return true;
3082 } else if (Name.starts_with("pmaddw.d.")) {
3083 if (VecWidth == 128)
3084 IID = Intrinsic::x86_sse2_pmadd_wd;
3085 else if (VecWidth == 256)
3086 IID = Intrinsic::x86_avx2_pmadd_wd;
3087 else if (VecWidth == 512)
3088 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3089 else
3090 llvm_unreachable("Unexpected intrinsic");
3091 } else if (Name.starts_with("pmaddubs.w.")) {
3092 if (VecWidth == 128)
3093 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3094 else if (VecWidth == 256)
3095 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3096 else if (VecWidth == 512)
3097 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3098 else
3099 llvm_unreachable("Unexpected intrinsic");
3100 } else if (Name.starts_with("packsswb.")) {
3101 if (VecWidth == 128)
3102 IID = Intrinsic::x86_sse2_packsswb_128;
3103 else if (VecWidth == 256)
3104 IID = Intrinsic::x86_avx2_packsswb;
3105 else if (VecWidth == 512)
3106 IID = Intrinsic::x86_avx512_packsswb_512;
3107 else
3108 llvm_unreachable("Unexpected intrinsic");
3109 } else if (Name.starts_with("packssdw.")) {
3110 if (VecWidth == 128)
3111 IID = Intrinsic::x86_sse2_packssdw_128;
3112 else if (VecWidth == 256)
3113 IID = Intrinsic::x86_avx2_packssdw;
3114 else if (VecWidth == 512)
3115 IID = Intrinsic::x86_avx512_packssdw_512;
3116 else
3117 llvm_unreachable("Unexpected intrinsic");
3118 } else if (Name.starts_with("packuswb.")) {
3119 if (VecWidth == 128)
3120 IID = Intrinsic::x86_sse2_packuswb_128;
3121 else if (VecWidth == 256)
3122 IID = Intrinsic::x86_avx2_packuswb;
3123 else if (VecWidth == 512)
3124 IID = Intrinsic::x86_avx512_packuswb_512;
3125 else
3126 llvm_unreachable("Unexpected intrinsic");
3127 } else if (Name.starts_with("packusdw.")) {
3128 if (VecWidth == 128)
3129 IID = Intrinsic::x86_sse41_packusdw;
3130 else if (VecWidth == 256)
3131 IID = Intrinsic::x86_avx2_packusdw;
3132 else if (VecWidth == 512)
3133 IID = Intrinsic::x86_avx512_packusdw_512;
3134 else
3135 llvm_unreachable("Unexpected intrinsic");
3136 } else if (Name.starts_with("vpermilvar.")) {
3137 if (VecWidth == 128 && EltWidth == 32)
3138 IID = Intrinsic::x86_avx_vpermilvar_ps;
3139 else if (VecWidth == 128 && EltWidth == 64)
3140 IID = Intrinsic::x86_avx_vpermilvar_pd;
3141 else if (VecWidth == 256 && EltWidth == 32)
3142 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3143 else if (VecWidth == 256 && EltWidth == 64)
3144 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3145 else if (VecWidth == 512 && EltWidth == 32)
3146 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3147 else if (VecWidth == 512 && EltWidth == 64)
3148 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3149 else
3150 llvm_unreachable("Unexpected intrinsic");
3151 } else if (Name == "cvtpd2dq.256") {
3152 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3153 } else if (Name == "cvtpd2ps.256") {
3154 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3155 } else if (Name == "cvttpd2dq.256") {
3156 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3157 } else if (Name == "cvttps2dq.128") {
3158 IID = Intrinsic::x86_sse2_cvttps2dq;
3159 } else if (Name == "cvttps2dq.256") {
3160 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3161 } else if (Name.starts_with("permvar.")) {
3162 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3163 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3164 IID = Intrinsic::x86_avx2_permps;
3165 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3166 IID = Intrinsic::x86_avx2_permd;
3167 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3168 IID = Intrinsic::x86_avx512_permvar_df_256;
3169 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3170 IID = Intrinsic::x86_avx512_permvar_di_256;
3171 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3172 IID = Intrinsic::x86_avx512_permvar_sf_512;
3173 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3174 IID = Intrinsic::x86_avx512_permvar_si_512;
3175 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3176 IID = Intrinsic::x86_avx512_permvar_df_512;
3177 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3178 IID = Intrinsic::x86_avx512_permvar_di_512;
3179 else if (VecWidth == 128 && EltWidth == 16)
3180 IID = Intrinsic::x86_avx512_permvar_hi_128;
3181 else if (VecWidth == 256 && EltWidth == 16)
3182 IID = Intrinsic::x86_avx512_permvar_hi_256;
3183 else if (VecWidth == 512 && EltWidth == 16)
3184 IID = Intrinsic::x86_avx512_permvar_hi_512;
3185 else if (VecWidth == 128 && EltWidth == 8)
3186 IID = Intrinsic::x86_avx512_permvar_qi_128;
3187 else if (VecWidth == 256 && EltWidth == 8)
3188 IID = Intrinsic::x86_avx512_permvar_qi_256;
3189 else if (VecWidth == 512 && EltWidth == 8)
3190 IID = Intrinsic::x86_avx512_permvar_qi_512;
3191 else
3192 llvm_unreachable("Unexpected intrinsic");
3193 } else if (Name.starts_with("dbpsadbw.")) {
3194 if (VecWidth == 128)
3195 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3196 else if (VecWidth == 256)
3197 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3198 else if (VecWidth == 512)
3199 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3200 else
3201 llvm_unreachable("Unexpected intrinsic");
3202 } else if (Name.starts_with("pmultishift.qb.")) {
3203 if (VecWidth == 128)
3204 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3205 else if (VecWidth == 256)
3206 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3207 else if (VecWidth == 512)
3208 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3209 else
3210 llvm_unreachable("Unexpected intrinsic");
3211 } else if (Name.starts_with("conflict.")) {
3212 if (Name[9] == 'd' && VecWidth == 128)
3213 IID = Intrinsic::x86_avx512_conflict_d_128;
3214 else if (Name[9] == 'd' && VecWidth == 256)
3215 IID = Intrinsic::x86_avx512_conflict_d_256;
3216 else if (Name[9] == 'd' && VecWidth == 512)
3217 IID = Intrinsic::x86_avx512_conflict_d_512;
3218 else if (Name[9] == 'q' && VecWidth == 128)
3219 IID = Intrinsic::x86_avx512_conflict_q_128;
3220 else if (Name[9] == 'q' && VecWidth == 256)
3221 IID = Intrinsic::x86_avx512_conflict_q_256;
3222 else if (Name[9] == 'q' && VecWidth == 512)
3223 IID = Intrinsic::x86_avx512_conflict_q_512;
3224 else
3225 llvm_unreachable("Unexpected intrinsic");
3226 } else if (Name.starts_with("pavg.")) {
3227 if (Name[5] == 'b' && VecWidth == 128)
3228 IID = Intrinsic::x86_sse2_pavg_b;
3229 else if (Name[5] == 'b' && VecWidth == 256)
3230 IID = Intrinsic::x86_avx2_pavg_b;
3231 else if (Name[5] == 'b' && VecWidth == 512)
3232 IID = Intrinsic::x86_avx512_pavg_b_512;
3233 else if (Name[5] == 'w' && VecWidth == 128)
3234 IID = Intrinsic::x86_sse2_pavg_w;
3235 else if (Name[5] == 'w' && VecWidth == 256)
3236 IID = Intrinsic::x86_avx2_pavg_w;
3237 else if (Name[5] == 'w' && VecWidth == 512)
3238 IID = Intrinsic::x86_avx512_pavg_w_512;
3239 else
3240 llvm_unreachable("Unexpected intrinsic");
3241 } else
3242 return false;
3243
3244 SmallVector<Value *, 4> Args(CI.args());
3245 Args.pop_back();
3246 Args.pop_back();
3247 Rep = Builder.CreateIntrinsic(IID, Args);
3248 unsigned NumArgs = CI.arg_size();
3249 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3250 CI.getArgOperand(NumArgs - 2));
3251 return true;
3252}
3253
3254/// Upgrade comment in call to inline asm that represents an objc retain release
3255/// marker.
3256void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3257 size_t Pos;
3258 if (AsmStr->find("mov\tfp") == 0 &&
3259 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3260 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3261 AsmStr->replace(Pos, 1, ";");
3262 }
3263}
3264
3265static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3266 StringRef Name,
3267 const Intrinsic::ID (&IIDs)[2][2]) {
3268 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3269 assert(Result && "unsupported nvvm.add.*/nvvm.mul.* intrinsic");
3270 auto [IID, RoundingMode] = *Result;
3271 Value *A = CI->getArgOperand(0);
3272 return Builder.CreateIntrinsic(
3273 A->getType(), IID,
3274 {A, CI->getArgOperand(1),
3275 Builder.getInt32(static_cast<int>(RoundingMode))});
3276}
3277
3278static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3279 Function *F, IRBuilder<> &Builder) {
3280 Value *Rep = nullptr;
3281
3282 if (Name == "abs.i" || Name == "abs.ll") {
3283 Value *Arg = CI->getArgOperand(0);
3284 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3285 {Arg, Builder.getTrue()},
3286 /*FMFSource=*/nullptr, "abs");
3287 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3288 Type *Ty = (Name == "abs.bf16")
3289 ? Builder.getBFloatTy()
3290 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3291 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3292 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3293 Rep = Builder.CreateBitCast(Abs, CI->getType());
3294 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3295 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3296 : Intrinsic::nvvm_fabs;
3297 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3298 } else if (Name.consume_front("add.")) {
3299 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3300 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFAddIIDs);
3301 } else if (Name.consume_front("mul.")) {
3302 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3303 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMulIIDs);
3304 } else if (Name.consume_front("ex2.approx.")) {
3305 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3306 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3307 : Intrinsic::nvvm_ex2_approx;
3308 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3309 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3310 Name.starts_with("atomic.load.add.f64.p")) {
3311 Value *Ptr = CI->getArgOperand(0);
3312 Value *Val = CI->getArgOperand(1);
3313 Rep = Builder.CreateAtomicRMW(
3314 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3315 CI->getContext().getOrInsertSyncScopeID("device"));
3316 cast<Instruction>(Rep)->setMetadata(
3317 LLVMContext::MD_atomic_ignore_denormal_mode,
3318 MDNode::get(CI->getContext(), {}));
3319 // The default scope for atomic.load.* intrinsics is device
3320 // (= gpu scope in ptx), but the default LLVM atomic scope is
3321 // "system"
3322 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3323 Name.starts_with("atomic.load.dec.32.p")) {
3324 Value *Ptr = CI->getArgOperand(0);
3325 Value *Val = CI->getArgOperand(1);
3326 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3327 : AtomicRMWInst::UDecWrap;
3328 Rep = Builder.CreateAtomicRMW(
3329 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3330 CI->getContext().getOrInsertSyncScopeID("device"));
3331 // See comment above.
3332 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3333 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3334 StringRef Op = Name.substr(StringRef("atomic.").size());
3335 Value *Ptr = CI->getArgOperand(0);
3336 Value *Val = CI->getArgOperand(1);
3337 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3338 Op.contains(".cta.") ? "block" : "");
3339 if (Op.starts_with("cas.")) {
3340 Value *New = CI->getArgOperand(2);
3341 Value *Pair = Builder.CreateAtomicCmpXchg(
3342 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3343 AtomicOrdering::Monotonic, SSID);
3344 Rep = Builder.CreateExtractValue(Pair, 0);
3345 } else {
3346 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3347 // because we were actually missing those intrinsics!
3348 AtomicRMWInst::BinOp BinOp =
3349 StringSwitch<AtomicRMWInst::BinOp>(Op)
3350 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3351 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3352 .StartsWith("exch.", AtomicRMWInst::Xchg)
3353 .StartsWith("max.", AtomicRMWInst::Max)
3354 .StartsWith("min.", AtomicRMWInst::Min)
3355 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3356 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3357 .StartsWith("and.", AtomicRMWInst::And)
3358 .StartsWith("or.", AtomicRMWInst::Or)
3359 .StartsWith("xor.", AtomicRMWInst::Xor)
3360 .Default(AtomicRMWInst::BAD_BINOP);
3361 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3362 "unexpected nvvm scoped atomic intrinsic");
3363 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3364 AtomicOrdering::Monotonic, SSID);
3365 if (BinOp == AtomicRMWInst::FAdd) {
3366 cast<Instruction>(Rep)->setMetadata(
3367 LLVMContext::MD_atomic_ignore_denormal_mode,
3368 MDNode::get(CI->getContext(), {}));
3369 }
3370 }
3371 } else if (Name == "clz.ll") {
3372 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3373 Value *Arg = CI->getArgOperand(0);
3374 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3375 {Arg, Builder.getFalse()},
3376 /*FMFSource=*/nullptr, "ctlz");
3377 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3378 } else if (Name == "popc.ll") {
3379 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3380 // i64.
3381 Value *Arg = CI->getArgOperand(0);
3382 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3383 Arg, /*FMFSource=*/nullptr, "ctpop");
3384 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3385 } else if (Name == "h2f") {
3386 Value *Cast =
3387 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3388 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3389 } else if (Name.consume_front("bitcast.") &&
3390 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3391 Name == "d2ll")) {
3392 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3393 } else if (Name == "rotate.b32") {
3394 Value *Arg = CI->getOperand(0);
3395 Value *ShiftAmt = CI->getOperand(1);
3396 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3397 {Arg, Arg, ShiftAmt});
3398 } else if (Name == "rotate.b64") {
3399 Type *Int64Ty = Builder.getInt64Ty();
3400 Value *Arg = CI->getOperand(0);
3401 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3402 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3403 {Arg, Arg, ZExtShiftAmt});
3404 } else if (Name == "rotate.right.b64") {
3405 Type *Int64Ty = Builder.getInt64Ty();
3406 Value *Arg = CI->getOperand(0);
3407 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3408 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3409 {Arg, Arg, ZExtShiftAmt});
3410 } else if (Name == "swap.lo.hi.b64") {
3411 Type *Int64Ty = Builder.getInt64Ty();
3412 Value *Arg = CI->getOperand(0);
3413 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3414 {Arg, Arg, Builder.getInt64(32)});
3415 } else if ((Name.consume_front("ptr.gen.to.") &&
3416 consumeNVVMPtrAddrSpace(Name)) ||
3417 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3418 Name.starts_with(".to.gen"))) {
3419 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3420 } else if (Name.consume_front("ldg.global")) {
3421 Value *Ptr = CI->getArgOperand(0);
3422 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3423 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3424 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3425 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3426 MDNode *MD = MDNode::get(Builder.getContext(), {});
3427 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3428 return LD;
3429 } else if (Name == "tanh.approx.f32") {
3430 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3431 FastMathFlags FMF;
3432 FMF.setApproxFunc();
3433 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3434 FMF);
3435 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3436 Value *Arg =
3437 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3438 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3439 {}, {Arg});
3440 } else if (Name == "barrier") {
3441 Rep = Builder.CreateIntrinsic(
3442 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3443 {CI->getArgOperand(0), CI->getArgOperand(1)});
3444 } else if (Name == "barrier.sync") {
3445 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3446 {CI->getArgOperand(0)});
3447 } else if (Name == "barrier.sync.cnt") {
3448 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3449 {CI->getArgOperand(0), CI->getArgOperand(1)});
3450 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3451 Name == "barrier0.or") {
3452 Value *C = CI->getArgOperand(0);
3453 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3454
3455 Intrinsic::ID IID =
3456 StringSwitch<Intrinsic::ID>(Name)
3457 .Case("barrier0.popc",
3458 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3459 .Case("barrier0.and",
3460 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3461 .Case("barrier0.or",
3462 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3463 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3464 Rep = Builder.CreateZExt(Bar, CI->getType());
3465 } else {
3466 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3467 if (IID != Intrinsic::not_intrinsic &&
3468 isLegacyNVPTXBF16IntSignature(F, IID)) {
3469 rename(F);
3470 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3471 SmallVector<Value *, 2> Args;
3472 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3473 Value *Arg = CI->getArgOperand(I);
3474 Type *OldType = Arg->getType();
3475 Type *NewType = NewFn->getArg(I)->getType();
3476 Args.push_back(
3477 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3478 ? Builder.CreateBitCast(Arg, NewType)
3479 : Arg);
3480 }
3481 Rep = Builder.CreateCall(NewFn, Args);
3482 if (F->getReturnType()->isIntegerTy())
3483 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3484 }
3485 }
3486
3487 return Rep;
3488}
3489
3490static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3491 IRBuilder<> &Builder) {
3492 LLVMContext &C = F->getContext();
3493 Value *Rep = nullptr;
3494
3495 if (Name.starts_with("sse4a.movnt.")) {
3496 SmallVector<Metadata *, 1> Elts;
3497 Elts.push_back(
3498 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3499 MDNode *Node = MDNode::get(C, Elts);
3500
3501 Value *Arg0 = CI->getArgOperand(0);
3502 Value *Arg1 = CI->getArgOperand(1);
3503
3504 // Nontemporal (unaligned) store of the 0'th element of the float/double
3505 // vector.
3506 Value *Extract =
3507 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3508
3509 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3510 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3511 } else if (Name.starts_with("avx.movnt.") ||
3512 Name.starts_with("avx512.storent.")) {
3513 SmallVector<Metadata *, 1> Elts;
3514 Elts.push_back(
3515 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3516 MDNode *Node = MDNode::get(C, Elts);
3517
3518 Value *Arg0 = CI->getArgOperand(0);
3519 Value *Arg1 = CI->getArgOperand(1);
3520
3521 StoreInst *SI = Builder.CreateAlignedStore(
3522 Arg1, Arg0,
3523 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3524 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3525 } else if (Name == "sse2.storel.dq") {
3526 Value *Arg0 = CI->getArgOperand(0);
3527 Value *Arg1 = CI->getArgOperand(1);
3528
3529 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3530 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3531 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3532 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3533 } else if (Name.starts_with("sse.storeu.") ||
3534 Name.starts_with("sse2.storeu.") ||
3535 Name.starts_with("avx.storeu.")) {
3536 Value *Arg0 = CI->getArgOperand(0);
3537 Value *Arg1 = CI->getArgOperand(1);
3538 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3539 } else if (Name == "avx512.mask.store.ss") {
3540 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3541 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3542 Mask, false);
3543 } else if (Name.starts_with("avx512.mask.store")) {
3544 // "avx512.mask.storeu." or "avx512.mask.store."
3545 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3546 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3547 CI->getArgOperand(2), Aligned);
3548 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3549 // Upgrade packed integer vector compare intrinsics to compare instructions.
3550 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3551 bool CmpEq = Name[9] == 'e';
3552 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3553 CI->getArgOperand(0), CI->getArgOperand(1));
3554 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3555 } else if (Name.starts_with("avx512.broadcastm")) {
3556 Type *ExtTy = Type::getInt32Ty(C);
3557 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3558 ExtTy = Type::getInt64Ty(C);
3559 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3560 ExtTy->getPrimitiveSizeInBits();
3561 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3562 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3563 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3564 Value *Vec = CI->getArgOperand(0);
3565 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3566 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3567 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3568 } else if (Name.starts_with("avx.sqrt.p") ||
3569 Name.starts_with("sse2.sqrt.p") ||
3570 Name.starts_with("sse.sqrt.p")) {
3571 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3572 {CI->getArgOperand(0)});
3573 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3574 if (CI->arg_size() == 4 &&
3575 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3576 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3577 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3578 : Intrinsic::x86_avx512_sqrt_pd_512;
3579
3580 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3581 Rep = Builder.CreateIntrinsic(IID, Args);
3582 } else {
3583 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3584 {CI->getArgOperand(0)});
3585 }
3586 Rep =
3587 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3588 } else if (Name.starts_with("avx512.ptestm") ||
3589 Name.starts_with("avx512.ptestnm")) {
3590 Value *Op0 = CI->getArgOperand(0);
3591 Value *Op1 = CI->getArgOperand(1);
3592 Value *Mask = CI->getArgOperand(2);
3593 Rep = Builder.CreateAnd(Op0, Op1);
3594 llvm::Type *Ty = Op0->getType();
3595 Value *Zero = llvm::Constant::getNullValue(Ty);
3596 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3597 ? ICmpInst::ICMP_NE
3598 : ICmpInst::ICMP_EQ;
3599 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3600 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3601 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3602 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3603 ->getNumElements();
3604 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3605 Rep =
3606 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3607 } else if (Name.starts_with("avx512.kunpck")) {
3608 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3609 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3610 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3611 int Indices[64];
3612 for (unsigned i = 0; i != NumElts; ++i)
3613 Indices[i] = i;
3614
3615 // First extract half of each vector. This gives better codegen than
3616 // doing it in a single shuffle.
3617 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3618 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3619 // Concat the vectors.
3620 // NOTE: Operands have to be swapped to match intrinsic definition.
3621 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3622 Rep = Builder.CreateBitCast(Rep, CI->getType());
3623 } else if (Name == "avx512.kand.w") {
3624 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3625 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3626 Rep = Builder.CreateAnd(LHS, RHS);
3627 Rep = Builder.CreateBitCast(Rep, CI->getType());
3628 } else if (Name == "avx512.kandn.w") {
3629 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3630 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3631 LHS = Builder.CreateNot(LHS);
3632 Rep = Builder.CreateAnd(LHS, RHS);
3633 Rep = Builder.CreateBitCast(Rep, CI->getType());
3634 } else if (Name == "avx512.kor.w") {
3635 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3636 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3637 Rep = Builder.CreateOr(LHS, RHS);
3638 Rep = Builder.CreateBitCast(Rep, CI->getType());
3639 } else if (Name == "avx512.kxor.w") {
3640 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3641 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3642 Rep = Builder.CreateXor(LHS, RHS);
3643 Rep = Builder.CreateBitCast(Rep, CI->getType());
3644 } else if (Name == "avx512.kxnor.w") {
3645 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3646 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3647 LHS = Builder.CreateNot(LHS);
3648 Rep = Builder.CreateXor(LHS, RHS);
3649 Rep = Builder.CreateBitCast(Rep, CI->getType());
3650 } else if (Name == "avx512.knot.w") {
3651 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3652 Rep = Builder.CreateNot(Rep);
3653 Rep = Builder.CreateBitCast(Rep, CI->getType());
3654 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3655 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3656 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3657 Rep = Builder.CreateOr(LHS, RHS);
3658 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3659 Value *C;
3660 if (Name[14] == 'c')
3661 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3662 else
3663 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3664 Rep = Builder.CreateICmpEQ(Rep, C);
3665 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3666 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3667 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3668 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3669 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3670 Type *I32Ty = Type::getInt32Ty(C);
3671 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3672 ConstantInt::get(I32Ty, 0));
3673 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3674 ConstantInt::get(I32Ty, 0));
3675 Value *EltOp;
3676 if (Name.contains(".add."))
3677 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3678 else if (Name.contains(".sub."))
3679 EltOp = Builder.CreateFSub(Elt0, Elt1);
3680 else if (Name.contains(".mul."))
3681 EltOp = Builder.CreateFMul(Elt0, Elt1);
3682 else
3683 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3684 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3685 ConstantInt::get(I32Ty, 0));
3686 } else if (Name.starts_with("avx512.mask.pcmp")) {
3687 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3688 bool CmpEq = Name[16] == 'e';
3689 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3690 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3691 Type *OpTy = CI->getArgOperand(0)->getType();
3692 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3693 Intrinsic::ID IID;
3694 switch (VecWidth) {
3695 default:
3696 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3697 break;
3698 case 128:
3699 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3700 break;
3701 case 256:
3702 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3703 break;
3704 case 512:
3705 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3706 break;
3707 }
3708
3709 Rep =
3710 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3711 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3712 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3713 Type *OpTy = CI->getArgOperand(0)->getType();
3714 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3715 unsigned EltWidth = OpTy->getScalarSizeInBits();
3716 Intrinsic::ID IID;
3717 if (VecWidth == 128 && EltWidth == 32)
3718 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3719 else if (VecWidth == 256 && EltWidth == 32)
3720 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3721 else if (VecWidth == 512 && EltWidth == 32)
3722 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3723 else if (VecWidth == 128 && EltWidth == 64)
3724 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3725 else if (VecWidth == 256 && EltWidth == 64)
3726 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3727 else if (VecWidth == 512 && EltWidth == 64)
3728 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3729 else
3730 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3731
3732 Rep =
3733 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3734 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3735 } else if (Name.starts_with("avx512.cmp.p")) {
3736 SmallVector<Value *, 4> Args(CI->args());
3737 Type *OpTy = Args[0]->getType();
3738 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3739 unsigned EltWidth = OpTy->getScalarSizeInBits();
3740 Intrinsic::ID IID;
3741 if (VecWidth == 128 && EltWidth == 32)
3742 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3743 else if (VecWidth == 256 && EltWidth == 32)
3744 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3745 else if (VecWidth == 512 && EltWidth == 32)
3746 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3747 else if (VecWidth == 128 && EltWidth == 64)
3748 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3749 else if (VecWidth == 256 && EltWidth == 64)
3750 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3751 else if (VecWidth == 512 && EltWidth == 64)
3752 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3753 else
3754 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3755
3756 Value *Mask = Constant::getAllOnesValue(CI->getType());
3757 if (VecWidth == 512)
3758 std::swap(Mask, Args.back());
3759 Args.push_back(Mask);
3760
3761 Rep = Builder.CreateIntrinsic(IID, Args);
3762 } else if (Name.starts_with("avx512.mask.cmp.")) {
3763 // Integer compare intrinsics.
3764 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3765 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3766 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3767 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3768 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3769 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3770 Name.starts_with("avx512.cvtw2mask.") ||
3771 Name.starts_with("avx512.cvtd2mask.") ||
3772 Name.starts_with("avx512.cvtq2mask.")) {
3773 Value *Op = CI->getArgOperand(0);
3774 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3775 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3776 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3777 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3778 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3779 Name.starts_with("avx512.mask.pabs")) {
3780 Rep = upgradeAbs(Builder, *CI);
3781 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3782 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3783 Name.starts_with("avx512.mask.pmaxs")) {
3784 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3785 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3786 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3787 Name.starts_with("avx512.mask.pmaxu")) {
3788 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3789 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3790 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3791 Name.starts_with("avx512.mask.pmins")) {
3792 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3793 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3794 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3795 Name.starts_with("avx512.mask.pminu")) {
3796 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3797 } else if (Name == "sse2.pmulh.w" || Name.starts_with("avx2.pmulh.w") ||
3798 Name.starts_with("avx512.pmulh.w")) {
3799 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smulh);
3800 } else if (Name == "sse2.pmulhu.w" || Name.starts_with("avx2.pmulhu.w") ||
3801 Name.starts_with("avx512.pmulhu.w")) {
3802 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umulh);
3803 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3804 Name == "avx512.pmulu.dq.512" ||
3805 Name.starts_with("avx512.mask.pmulu.dq.")) {
3806 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3807 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3808 Name == "avx512.pmul.dq.512" ||
3809 Name.starts_with("avx512.mask.pmul.dq.")) {
3810 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3811 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3812 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3813 Rep =
3814 Builder.CreateSIToFP(CI->getArgOperand(1),
3815 cast<VectorType>(CI->getType())->getElementType());
3816 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3817 } else if (Name == "avx512.cvtusi2sd") {
3818 Rep =
3819 Builder.CreateUIToFP(CI->getArgOperand(1),
3820 cast<VectorType>(CI->getType())->getElementType());
3821 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3822 } else if (Name == "sse2.cvtss2sd") {
3823 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3824 Rep = Builder.CreateFPExt(
3825 Rep, cast<VectorType>(CI->getType())->getElementType());
3826 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3827 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3828 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3829 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3830 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3831 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3832 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3833 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3834 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3835 Name == "avx512.mask.cvtqq2ps.256" ||
3836 Name == "avx512.mask.cvtqq2ps.512" ||
3837 Name == "avx512.mask.cvtuqq2ps.256" ||
3838 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3839 Name == "avx.cvt.ps2.pd.256" ||
3840 Name == "avx512.mask.cvtps2pd.128" ||
3841 Name == "avx512.mask.cvtps2pd.256") {
3842 auto *DstTy = cast<FixedVectorType>(CI->getType());
3843 Rep = CI->getArgOperand(0);
3844 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3845
3846 unsigned NumDstElts = DstTy->getNumElements();
3847 if (NumDstElts < SrcTy->getNumElements()) {
3848 assert(NumDstElts == 2 && "Unexpected vector size");
3849 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3850 }
3851
3852 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3853 bool IsUnsigned = Name.contains("cvtu");
3854 if (IsPS2PD)
3855 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3856 else if (CI->arg_size() == 4 &&
3857 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3858 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3859 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3860 : Intrinsic::x86_avx512_sitofp_round;
3861 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3862 {Rep, CI->getArgOperand(3)});
3863 } else {
3864 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3865 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3866 }
3867
3868 if (CI->arg_size() >= 3)
3869 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3870 CI->getArgOperand(1));
3871 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3872 Name.starts_with("vcvtph2ps.")) {
3873 auto *DstTy = cast<FixedVectorType>(CI->getType());
3874 Rep = CI->getArgOperand(0);
3875 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3876 unsigned NumDstElts = DstTy->getNumElements();
3877 if (NumDstElts != SrcTy->getNumElements()) {
3878 assert(NumDstElts == 4 && "Unexpected vector size");
3879 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3880 }
3881 Rep = Builder.CreateBitCast(
3882 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3883 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3884 if (CI->arg_size() >= 3)
3885 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3886 CI->getArgOperand(1));
3887 } else if (Name.starts_with("avx512.mask.load")) {
3888 // "avx512.mask.loadu." or "avx512.mask.load."
3889 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3890 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3891 CI->getArgOperand(2), Aligned);
3892 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3893 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3894 auto *PtrTy = CI->getOperand(0)->getType();
3895 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3896 ResultTy->getNumElements());
3897 Rep = Builder.CreateIntrinsic(
3898 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3899 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3900 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3901 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3902 auto *PtrTy = CI->getArgOperand(0)->getType();
3903 Value *MaskVec =
3904 getX86MaskVec(Builder, CI->getArgOperand(2),
3905 cast<FixedVectorType>(ResultTy)->getNumElements());
3906 Rep = Builder.CreateIntrinsic(
3907 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3908 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3909 } else if (Name.starts_with("avx512.mask.compress.") ||
3910 Name.starts_with("avx512.mask.expand.")) {
3911 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3912
3913 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3914 ResultTy->getNumElements());
3915
3916 bool IsCompress = Name[12] == 'c';
3917 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3918 : Intrinsic::x86_avx512_mask_expand;
3919 Rep = Builder.CreateIntrinsic(
3920 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3921 } else if (Name.starts_with("xop.vpcom")) {
3922 bool IsSigned;
3923 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3924 Name.ends_with("uq"))
3925 IsSigned = false;
3926 else if (Name.ends_with("b") || Name.ends_with("w") ||
3927 Name.ends_with("d") || Name.ends_with("q"))
3928 IsSigned = true;
3929 else
3930 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3931
3932 unsigned Imm;
3933 if (CI->arg_size() == 3) {
3934 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3935 } else {
3936 Name = Name.substr(9); // strip off "xop.vpcom"
3937 if (Name.starts_with("lt"))
3938 Imm = 0;
3939 else if (Name.starts_with("le"))
3940 Imm = 1;
3941 else if (Name.starts_with("gt"))
3942 Imm = 2;
3943 else if (Name.starts_with("ge"))
3944 Imm = 3;
3945 else if (Name.starts_with("eq"))
3946 Imm = 4;
3947 else if (Name.starts_with("ne"))
3948 Imm = 5;
3949 else if (Name.starts_with("false"))
3950 Imm = 6;
3951 else if (Name.starts_with("true"))
3952 Imm = 7;
3953 else
3954 llvm_unreachable("Unknown condition");
3955 }
3956
3957 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3958 } else if (Name.starts_with("xop.vpcmov")) {
3959 Value *Sel = CI->getArgOperand(2);
3960 Value *NotSel = Builder.CreateNot(Sel);
3961 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3962 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3963 Rep = Builder.CreateOr(Sel0, Sel1);
3964 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3965 Name.starts_with("avx512.mask.prol")) {
3966 Rep = upgradeX86Rotate(Builder, *CI, false);
3967 } else if (Name.starts_with("avx512.pror") ||
3968 Name.starts_with("avx512.mask.pror")) {
3969 Rep = upgradeX86Rotate(Builder, *CI, true);
3970 } else if (Name.starts_with("avx512.vpshld.") ||
3971 Name.starts_with("avx512.mask.vpshld") ||
3972 Name.starts_with("avx512.maskz.vpshld")) {
3973 bool ZeroMask = Name[11] == 'z';
3974 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3975 } else if (Name.starts_with("avx512.vpshrd.") ||
3976 Name.starts_with("avx512.mask.vpshrd") ||
3977 Name.starts_with("avx512.maskz.vpshrd")) {
3978 bool ZeroMask = Name[11] == 'z';
3979 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3980 } else if (Name == "sse42.crc32.64.8") {
3981 Value *Trunc0 =
3982 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3983 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3984 {Trunc0, CI->getArgOperand(1)});
3985 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3986 } else if (Name.starts_with("avx.vbroadcast.s") ||
3987 Name.starts_with("avx512.vbroadcast.s")) {
3988 // Replace broadcasts with a series of insertelements.
3989 auto *VecTy = cast<FixedVectorType>(CI->getType());
3990 Type *EltTy = VecTy->getElementType();
3991 unsigned EltNum = VecTy->getNumElements();
3992 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3993 Type *I32Ty = Type::getInt32Ty(C);
3994 Rep = PoisonValue::get(VecTy);
3995 for (unsigned I = 0; I < EltNum; ++I)
3996 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3997 } else if (Name.starts_with("sse41.pmovsx") ||
3998 Name.starts_with("sse41.pmovzx") ||
3999 Name.starts_with("avx2.pmovsx") ||
4000 Name.starts_with("avx2.pmovzx") ||
4001 Name.starts_with("avx512.mask.pmovsx") ||
4002 Name.starts_with("avx512.mask.pmovzx")) {
4003 auto *DstTy = cast<FixedVectorType>(CI->getType());
4004 unsigned NumDstElts = DstTy->getNumElements();
4005
4006 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4007 SmallVector<int, 8> ShuffleMask(NumDstElts);
4008 for (unsigned i = 0; i != NumDstElts; ++i)
4009 ShuffleMask[i] = i;
4010
4011 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
4012
4013 bool DoSext = Name.contains("pmovsx");
4014 Rep =
4015 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4016 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4017 if (CI->arg_size() == 3)
4018 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4019 CI->getArgOperand(1));
4020 } else if (Name == "avx512.mask.pmov.qd.256" ||
4021 Name == "avx512.mask.pmov.qd.512" ||
4022 Name == "avx512.mask.pmov.wb.256" ||
4023 Name == "avx512.mask.pmov.wb.512") {
4024 Type *Ty = CI->getArgOperand(1)->getType();
4025 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
4026 Rep =
4027 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4028 } else if (Name.starts_with("avx.vbroadcastf128") ||
4029 Name == "avx2.vbroadcasti128") {
4030 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4031 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4032 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4033 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4034 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4035 if (NumSrcElts == 2)
4036 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4037 else
4038 Rep = Builder.CreateShuffleVector(Load,
4039 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4040 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4041 Name.starts_with("avx512.mask.shuf.f")) {
4042 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4043 Type *VT = CI->getType();
4044 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4045 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4046 unsigned ControlBitsMask = NumLanes - 1;
4047 unsigned NumControlBits = NumLanes / 2;
4048 SmallVector<int, 8> ShuffleMask(0);
4049
4050 for (unsigned l = 0; l != NumLanes; ++l) {
4051 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4052 // We actually need the other source.
4053 if (l >= NumLanes / 2)
4054 LaneMask += NumLanes;
4055 for (unsigned i = 0; i != NumElementsInLane; ++i)
4056 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4057 }
4058 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4059 CI->getArgOperand(1), ShuffleMask);
4060 Rep =
4061 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4062 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4063 Name.starts_with("avx512.mask.broadcasti")) {
4064 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4065 ->getNumElements();
4066 unsigned NumDstElts =
4067 cast<FixedVectorType>(CI->getType())->getNumElements();
4068
4069 SmallVector<int, 8> ShuffleMask(NumDstElts);
4070 for (unsigned i = 0; i != NumDstElts; ++i)
4071 ShuffleMask[i] = i % NumSrcElts;
4072
4073 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4074 CI->getArgOperand(0), ShuffleMask);
4075 Rep =
4076 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4077 } else if (Name.starts_with("avx2.pbroadcast") ||
4078 Name.starts_with("avx2.vbroadcast") ||
4079 Name.starts_with("avx512.pbroadcast") ||
4080 Name.starts_with("avx512.mask.broadcast.s")) {
4081 // Replace vp?broadcasts with a vector shuffle.
4082 Value *Op = CI->getArgOperand(0);
4083 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4084 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4085 SmallVector<int, 8> M;
4086 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4087 Rep = Builder.CreateShuffleVector(Op, M);
4088
4089 if (CI->arg_size() == 3)
4090 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4091 CI->getArgOperand(1));
4092 } else if (Name.starts_with("sse2.padds.") ||
4093 Name.starts_with("avx2.padds.") ||
4094 Name.starts_with("avx512.padds.") ||
4095 Name.starts_with("avx512.mask.padds.")) {
4096 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4097 } else if (Name.starts_with("sse2.psubs.") ||
4098 Name.starts_with("avx2.psubs.") ||
4099 Name.starts_with("avx512.psubs.") ||
4100 Name.starts_with("avx512.mask.psubs.")) {
4101 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4102 } else if (Name.starts_with("sse2.paddus.") ||
4103 Name.starts_with("avx2.paddus.") ||
4104 Name.starts_with("avx512.mask.paddus.")) {
4105 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4106 } else if (Name.starts_with("sse2.psubus.") ||
4107 Name.starts_with("avx2.psubus.") ||
4108 Name.starts_with("avx512.mask.psubus.")) {
4109 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4110 } else if (Name.starts_with("avx512.mask.palignr.")) {
4111 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4112 CI->getArgOperand(1), CI->getArgOperand(2),
4113 CI->getArgOperand(3), CI->getArgOperand(4),
4114 false);
4115 } else if (Name.starts_with("avx512.mask.valign.")) {
4116 Rep = upgradeX86ALIGNIntrinsics(
4117 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4118 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4119 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4120 // 128/256-bit shift left specified in bits.
4121 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4122 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4123 Shift / 8); // Shift is in bits.
4124 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4125 // 128/256-bit shift right specified in bits.
4126 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4127 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4128 Shift / 8); // Shift is in bits.
4129 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4130 Name == "avx512.psll.dq.512") {
4131 // 128/256/512-bit shift left specified in bytes.
4132 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4133 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4134 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4135 Name == "avx512.psrl.dq.512") {
4136 // 128/256/512-bit shift right specified in bytes.
4137 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4138 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4139 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4140 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4141 Name.starts_with("avx2.pblendd.")) {
4142 Value *Op0 = CI->getArgOperand(0);
4143 Value *Op1 = CI->getArgOperand(1);
4144 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4145 auto *VecTy = cast<FixedVectorType>(CI->getType());
4146 unsigned NumElts = VecTy->getNumElements();
4147
4148 SmallVector<int, 16> Idxs(NumElts);
4149 for (unsigned i = 0; i != NumElts; ++i)
4150 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4151
4152 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4153 } else if (Name.starts_with("avx.vinsertf128.") ||
4154 Name == "avx2.vinserti128" ||
4155 Name.starts_with("avx512.mask.insert")) {
4156 Value *Op0 = CI->getArgOperand(0);
4157 Value *Op1 = CI->getArgOperand(1);
4158 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4159 unsigned DstNumElts =
4160 cast<FixedVectorType>(CI->getType())->getNumElements();
4161 unsigned SrcNumElts =
4162 cast<FixedVectorType>(Op1->getType())->getNumElements();
4163 unsigned Scale = DstNumElts / SrcNumElts;
4164
4165 // Mask off the high bits of the immediate value; hardware ignores those.
4166 Imm = Imm % Scale;
4167
4168 // Extend the second operand into a vector the size of the destination.
4169 SmallVector<int, 8> Idxs(DstNumElts);
4170 for (unsigned i = 0; i != SrcNumElts; ++i)
4171 Idxs[i] = i;
4172 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4173 Idxs[i] = SrcNumElts;
4174 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4175
4176 // Insert the second operand into the first operand.
4177
4178 // Note that there is no guarantee that instruction lowering will actually
4179 // produce a vinsertf128 instruction for the created shuffles. In
4180 // particular, the 0 immediate case involves no lane changes, so it can
4181 // be handled as a blend.
4182
4183 // Example of shuffle mask for 32-bit elements:
4184 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4185 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4186
4187 // First fill with identify mask.
4188 for (unsigned i = 0; i != DstNumElts; ++i)
4189 Idxs[i] = i;
4190 // Then replace the elements where we need to insert.
4191 for (unsigned i = 0; i != SrcNumElts; ++i)
4192 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4193 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4194
4195 // If the intrinsic has a mask operand, handle that.
4196 if (CI->arg_size() == 5)
4197 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4198 CI->getArgOperand(3));
4199 } else if (Name.starts_with("avx.vextractf128.") ||
4200 Name == "avx2.vextracti128" ||
4201 Name.starts_with("avx512.mask.vextract")) {
4202 Value *Op0 = CI->getArgOperand(0);
4203 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4204 unsigned DstNumElts =
4205 cast<FixedVectorType>(CI->getType())->getNumElements();
4206 unsigned SrcNumElts =
4207 cast<FixedVectorType>(Op0->getType())->getNumElements();
4208 unsigned Scale = SrcNumElts / DstNumElts;
4209
4210 // Mask off the high bits of the immediate value; hardware ignores those.
4211 Imm = Imm % Scale;
4212
4213 // Get indexes for the subvector of the input vector.
4214 SmallVector<int, 8> Idxs(DstNumElts);
4215 for (unsigned i = 0; i != DstNumElts; ++i) {
4216 Idxs[i] = i + (Imm * DstNumElts);
4217 }
4218 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4219
4220 // If the intrinsic has a mask operand, handle that.
4221 if (CI->arg_size() == 4)
4222 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4223 CI->getArgOperand(2));
4224 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4225 Name.starts_with("avx512.mask.perm.di.")) {
4226 Value *Op0 = CI->getArgOperand(0);
4227 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4228 auto *VecTy = cast<FixedVectorType>(CI->getType());
4229 unsigned NumElts = VecTy->getNumElements();
4230
4231 SmallVector<int, 8> Idxs(NumElts);
4232 for (unsigned i = 0; i != NumElts; ++i)
4233 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4234
4235 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4236
4237 if (CI->arg_size() == 4)
4238 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4239 CI->getArgOperand(2));
4240 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4241 // The immediate permute control byte looks like this:
4242 // [1:0] - select 128 bits from sources for low half of destination
4243 // [2] - ignore
4244 // [3] - zero low half of destination
4245 // [5:4] - select 128 bits from sources for high half of destination
4246 // [6] - ignore
4247 // [7] - zero high half of destination
4248
4249 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4250
4251 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4252 unsigned HalfSize = NumElts / 2;
4253 SmallVector<int, 8> ShuffleMask(NumElts);
4254
4255 // Determine which operand(s) are actually in use for this instruction.
4256 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4257 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4258
4259 // If needed, replace operands based on zero mask.
4260 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4261 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4262
4263 // Permute low half of result.
4264 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4265 for (unsigned i = 0; i < HalfSize; ++i)
4266 ShuffleMask[i] = StartIndex + i;
4267
4268 // Permute high half of result.
4269 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4270 for (unsigned i = 0; i < HalfSize; ++i)
4271 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4272
4273 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4274
4275 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4276 Name.starts_with("avx512.mask.vpermil.p") ||
4277 Name.starts_with("avx512.mask.pshuf.d.")) {
4278 Value *Op0 = CI->getArgOperand(0);
4279 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4280 auto *VecTy = cast<FixedVectorType>(CI->getType());
4281 unsigned NumElts = VecTy->getNumElements();
4282 // Calculate the size of each index in the immediate.
4283 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4284 unsigned IdxMask = ((1 << IdxSize) - 1);
4285
4286 SmallVector<int, 8> Idxs(NumElts);
4287 // Lookup the bits for this element, wrapping around the immediate every
4288 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4289 // to offset by the first index of each group.
4290 for (unsigned i = 0; i != NumElts; ++i)
4291 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4292
4293 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4294
4295 if (CI->arg_size() == 4)
4296 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4297 CI->getArgOperand(2));
4298 } else if (Name == "sse2.pshufl.w" ||
4299 Name.starts_with("avx512.mask.pshufl.w.")) {
4300 Value *Op0 = CI->getArgOperand(0);
4301 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4302 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4303
4304 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4305 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4306
4307 SmallVector<int, 16> Idxs(NumElts);
4308 for (unsigned l = 0; l != NumElts; l += 8) {
4309 for (unsigned i = 0; i != 4; ++i)
4310 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4311 for (unsigned i = 4; i != 8; ++i)
4312 Idxs[i + l] = i + l;
4313 }
4314
4315 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4316
4317 if (CI->arg_size() == 4)
4318 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4319 CI->getArgOperand(2));
4320 } else if (Name == "sse2.pshufh.w" ||
4321 Name.starts_with("avx512.mask.pshufh.w.")) {
4322 Value *Op0 = CI->getArgOperand(0);
4323 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4324 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4325
4326 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4327 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4328
4329 SmallVector<int, 16> Idxs(NumElts);
4330 for (unsigned l = 0; l != NumElts; l += 8) {
4331 for (unsigned i = 0; i != 4; ++i)
4332 Idxs[i + l] = i + l;
4333 for (unsigned i = 0; i != 4; ++i)
4334 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4335 }
4336
4337 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4338
4339 if (CI->arg_size() == 4)
4340 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4341 CI->getArgOperand(2));
4342 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4343 Value *Op0 = CI->getArgOperand(0);
4344 Value *Op1 = CI->getArgOperand(1);
4345 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4346 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4347
4348 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4349 unsigned HalfLaneElts = NumLaneElts / 2;
4350
4351 SmallVector<int, 16> Idxs(NumElts);
4352 for (unsigned i = 0; i != NumElts; ++i) {
4353 // Base index is the starting element of the lane.
4354 Idxs[i] = i - (i % NumLaneElts);
4355 // If we are half way through the lane switch to the other source.
4356 if ((i % NumLaneElts) >= HalfLaneElts)
4357 Idxs[i] += NumElts;
4358 // Now select the specific element. By adding HalfLaneElts bits from
4359 // the immediate. Wrapping around the immediate every 8-bits.
4360 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4361 }
4362
4363 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4364
4365 Rep =
4366 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4367 } else if (Name.starts_with("avx512.mask.movddup") ||
4368 Name.starts_with("avx512.mask.movshdup") ||
4369 Name.starts_with("avx512.mask.movsldup")) {
4370 Value *Op0 = CI->getArgOperand(0);
4371 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4372 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4373
4374 unsigned Offset = 0;
4375 if (Name.starts_with("avx512.mask.movshdup."))
4376 Offset = 1;
4377
4378 SmallVector<int, 16> Idxs(NumElts);
4379 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4380 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4381 Idxs[i + l + 0] = i + l + Offset;
4382 Idxs[i + l + 1] = i + l + Offset;
4383 }
4384
4385 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4386
4387 Rep =
4388 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4389 } else if (Name.starts_with("avx512.mask.punpckl") ||
4390 Name.starts_with("avx512.mask.unpckl.")) {
4391 Value *Op0 = CI->getArgOperand(0);
4392 Value *Op1 = CI->getArgOperand(1);
4393 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4394 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4395
4396 SmallVector<int, 64> Idxs(NumElts);
4397 for (int l = 0; l != NumElts; l += NumLaneElts)
4398 for (int i = 0; i != NumLaneElts; ++i)
4399 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4400
4401 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4402
4403 Rep =
4404 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4405 } else if (Name.starts_with("avx512.mask.punpckh") ||
4406 Name.starts_with("avx512.mask.unpckh.")) {
4407 Value *Op0 = CI->getArgOperand(0);
4408 Value *Op1 = CI->getArgOperand(1);
4409 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4410 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4411
4412 SmallVector<int, 64> Idxs(NumElts);
4413 for (int l = 0; l != NumElts; l += NumLaneElts)
4414 for (int i = 0; i != NumLaneElts; ++i)
4415 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4416
4417 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4418
4419 Rep =
4420 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4421 } else if (Name.starts_with("avx512.mask.and.") ||
4422 Name.starts_with("avx512.mask.pand.")) {
4423 VectorType *FTy = cast<VectorType>(CI->getType());
4424 VectorType *ITy = VectorType::getInteger(FTy);
4425 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4426 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4427 Rep = Builder.CreateBitCast(Rep, FTy);
4428 Rep =
4429 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4430 } else if (Name.starts_with("avx512.mask.andn.") ||
4431 Name.starts_with("avx512.mask.pandn.")) {
4432 VectorType *FTy = cast<VectorType>(CI->getType());
4433 VectorType *ITy = VectorType::getInteger(FTy);
4434 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4435 Rep = Builder.CreateAnd(Rep,
4436 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4437 Rep = Builder.CreateBitCast(Rep, FTy);
4438 Rep =
4439 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4440 } else if (Name.starts_with("avx512.mask.or.") ||
4441 Name.starts_with("avx512.mask.por.")) {
4442 VectorType *FTy = cast<VectorType>(CI->getType());
4443 VectorType *ITy = VectorType::getInteger(FTy);
4444 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4445 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4446 Rep = Builder.CreateBitCast(Rep, FTy);
4447 Rep =
4448 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4449 } else if (Name.starts_with("avx512.mask.xor.") ||
4450 Name.starts_with("avx512.mask.pxor.")) {
4451 VectorType *FTy = cast<VectorType>(CI->getType());
4452 VectorType *ITy = VectorType::getInteger(FTy);
4453 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4454 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4455 Rep = Builder.CreateBitCast(Rep, FTy);
4456 Rep =
4457 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4458 } else if (Name.starts_with("avx512.mask.padd.")) {
4459 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4460 Rep =
4461 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4462 } else if (Name.starts_with("avx512.mask.psub.")) {
4463 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4464 Rep =
4465 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4466 } else if (Name.starts_with("avx512.mask.pmull.")) {
4467 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4468 Rep =
4469 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4470 } else if (Name.starts_with("avx512.mask.add.p")) {
4471 if (Name.ends_with(".512")) {
4472 Intrinsic::ID IID;
4473 if (Name[17] == 's')
4474 IID = Intrinsic::x86_avx512_add_ps_512;
4475 else
4476 IID = Intrinsic::x86_avx512_add_pd_512;
4477
4478 Rep = Builder.CreateIntrinsic(
4479 IID,
4480 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4481 } else {
4482 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4483 }
4484 Rep =
4485 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4486 } else if (Name.starts_with("avx512.mask.div.p")) {
4487 if (Name.ends_with(".512")) {
4488 Intrinsic::ID IID;
4489 if (Name[17] == 's')
4490 IID = Intrinsic::x86_avx512_div_ps_512;
4491 else
4492 IID = Intrinsic::x86_avx512_div_pd_512;
4493
4494 Rep = Builder.CreateIntrinsic(
4495 IID,
4496 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4497 } else {
4498 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4499 }
4500 Rep =
4501 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4502 } else if (Name.starts_with("avx512.mask.mul.p")) {
4503 if (Name.ends_with(".512")) {
4504 Intrinsic::ID IID;
4505 if (Name[17] == 's')
4506 IID = Intrinsic::x86_avx512_mul_ps_512;
4507 else
4508 IID = Intrinsic::x86_avx512_mul_pd_512;
4509
4510 Rep = Builder.CreateIntrinsic(
4511 IID,
4512 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4513 } else {
4514 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4515 }
4516 Rep =
4517 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4518 } else if (Name.starts_with("avx512.mask.sub.p")) {
4519 if (Name.ends_with(".512")) {
4520 Intrinsic::ID IID;
4521 if (Name[17] == 's')
4522 IID = Intrinsic::x86_avx512_sub_ps_512;
4523 else
4524 IID = Intrinsic::x86_avx512_sub_pd_512;
4525
4526 Rep = Builder.CreateIntrinsic(
4527 IID,
4528 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4529 } else {
4530 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4531 }
4532 Rep =
4533 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4534 } else if ((Name.starts_with("avx512.mask.max.p") ||
4535 Name.starts_with("avx512.mask.min.p")) &&
4536 Name.drop_front(18) == ".512") {
4537 bool IsDouble = Name[17] == 'd';
4538 bool IsMin = Name[13] == 'i';
4539 static const Intrinsic::ID MinMaxTbl[2][2] = {
4540 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4541 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4542 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4543
4544 Rep = Builder.CreateIntrinsic(
4545 IID,
4546 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4547 Rep =
4548 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4549 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4550 Rep =
4551 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4552 {CI->getArgOperand(0), Builder.getInt1(false)});
4553 Rep =
4554 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4555 } else if (Name.starts_with("avx512.mask.psll")) {
4556 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4557 bool IsVariable = Name[16] == 'v';
4558 char Size = Name[16] == '.' ? Name[17]
4559 : Name[17] == '.' ? Name[18]
4560 : Name[18] == '.' ? Name[19]
4561 : Name[20];
4562
4563 Intrinsic::ID IID;
4564 if (IsVariable && Name[17] != '.') {
4565 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4566 IID = Intrinsic::x86_avx2_psllv_q;
4567 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4568 IID = Intrinsic::x86_avx2_psllv_q_256;
4569 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4570 IID = Intrinsic::x86_avx2_psllv_d;
4571 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4572 IID = Intrinsic::x86_avx2_psllv_d_256;
4573 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4574 IID = Intrinsic::x86_avx512_psllv_w_128;
4575 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4576 IID = Intrinsic::x86_avx512_psllv_w_256;
4577 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4578 IID = Intrinsic::x86_avx512_psllv_w_512;
4579 else
4580 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4581 } else if (Name.ends_with(".128")) {
4582 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4583 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4584 : Intrinsic::x86_sse2_psll_d;
4585 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4586 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4587 : Intrinsic::x86_sse2_psll_q;
4588 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4589 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4590 : Intrinsic::x86_sse2_psll_w;
4591 else
4592 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4593 } else if (Name.ends_with(".256")) {
4594 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4595 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4596 : Intrinsic::x86_avx2_psll_d;
4597 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4598 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4599 : Intrinsic::x86_avx2_psll_q;
4600 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4601 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4602 : Intrinsic::x86_avx2_psll_w;
4603 else
4604 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4605 } else {
4606 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4607 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4608 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4609 : Intrinsic::x86_avx512_psll_d_512;
4610 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4611 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4612 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4613 : Intrinsic::x86_avx512_psll_q_512;
4614 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4615 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4616 : Intrinsic::x86_avx512_psll_w_512;
4617 else
4618 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4619 }
4620
4621 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4622 } else if (Name.starts_with("avx512.mask.psrl")) {
4623 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4624 bool IsVariable = Name[16] == 'v';
4625 char Size = Name[16] == '.' ? Name[17]
4626 : Name[17] == '.' ? Name[18]
4627 : Name[18] == '.' ? Name[19]
4628 : Name[20];
4629
4630 Intrinsic::ID IID;
4631 if (IsVariable && Name[17] != '.') {
4632 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4633 IID = Intrinsic::x86_avx2_psrlv_q;
4634 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4635 IID = Intrinsic::x86_avx2_psrlv_q_256;
4636 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4637 IID = Intrinsic::x86_avx2_psrlv_d;
4638 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4639 IID = Intrinsic::x86_avx2_psrlv_d_256;
4640 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4641 IID = Intrinsic::x86_avx512_psrlv_w_128;
4642 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4643 IID = Intrinsic::x86_avx512_psrlv_w_256;
4644 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4645 IID = Intrinsic::x86_avx512_psrlv_w_512;
4646 else
4647 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4648 } else if (Name.ends_with(".128")) {
4649 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4650 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4651 : Intrinsic::x86_sse2_psrl_d;
4652 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4653 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4654 : Intrinsic::x86_sse2_psrl_q;
4655 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4656 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4657 : Intrinsic::x86_sse2_psrl_w;
4658 else
4659 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4660 } else if (Name.ends_with(".256")) {
4661 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4662 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4663 : Intrinsic::x86_avx2_psrl_d;
4664 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4665 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4666 : Intrinsic::x86_avx2_psrl_q;
4667 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4668 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4669 : Intrinsic::x86_avx2_psrl_w;
4670 else
4671 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4672 } else {
4673 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4674 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4675 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4676 : Intrinsic::x86_avx512_psrl_d_512;
4677 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4678 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4679 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4680 : Intrinsic::x86_avx512_psrl_q_512;
4681 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4682 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4683 : Intrinsic::x86_avx512_psrl_w_512;
4684 else
4685 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4686 }
4687
4688 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4689 } else if (Name.starts_with("avx512.mask.psra")) {
4690 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4691 bool IsVariable = Name[16] == 'v';
4692 char Size = Name[16] == '.' ? Name[17]
4693 : Name[17] == '.' ? Name[18]
4694 : Name[18] == '.' ? Name[19]
4695 : Name[20];
4696
4697 Intrinsic::ID IID;
4698 if (IsVariable && Name[17] != '.') {
4699 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4700 IID = Intrinsic::x86_avx2_psrav_d;
4701 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4702 IID = Intrinsic::x86_avx2_psrav_d_256;
4703 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4704 IID = Intrinsic::x86_avx512_psrav_w_128;
4705 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4706 IID = Intrinsic::x86_avx512_psrav_w_256;
4707 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4708 IID = Intrinsic::x86_avx512_psrav_w_512;
4709 else
4710 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4711 } else if (Name.ends_with(".128")) {
4712 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4713 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4714 : Intrinsic::x86_sse2_psra_d;
4715 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4716 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4717 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4718 : Intrinsic::x86_avx512_psra_q_128;
4719 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4720 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4721 : Intrinsic::x86_sse2_psra_w;
4722 else
4723 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4724 } else if (Name.ends_with(".256")) {
4725 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4726 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4727 : Intrinsic::x86_avx2_psra_d;
4728 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4729 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4730 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4731 : Intrinsic::x86_avx512_psra_q_256;
4732 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4733 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4734 : Intrinsic::x86_avx2_psra_w;
4735 else
4736 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4737 } else {
4738 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4739 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4740 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4741 : Intrinsic::x86_avx512_psra_d_512;
4742 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4743 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4744 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4745 : Intrinsic::x86_avx512_psra_q_512;
4746 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4747 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4748 : Intrinsic::x86_avx512_psra_w_512;
4749 else
4750 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4751 }
4752
4753 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4754 } else if (Name.starts_with("avx512.mask.move.s")) {
4755 Rep = upgradeMaskedMove(Builder, *CI);
4756 } else if (Name.starts_with("avx512.cvtmask2")) {
4757 Rep = upgradeMaskToInt(Builder, *CI);
4758 } else if (Name.ends_with(".movntdqa")) {
4759 MDNode *Node = MDNode::get(
4760 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4761
4762 LoadInst *LI = Builder.CreateAlignedLoad(
4763 CI->getType(), CI->getArgOperand(0),
4764 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4765 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4766 Rep = LI;
4767 } else if (Name.starts_with("fma.vfmadd.") ||
4768 Name.starts_with("fma.vfmsub.") ||
4769 Name.starts_with("fma.vfnmadd.") ||
4770 Name.starts_with("fma.vfnmsub.")) {
4771 bool NegMul = Name[6] == 'n';
4772 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4773 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4774
4775 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4776 CI->getArgOperand(2)};
4777
4778 if (IsScalar) {
4779 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4780 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4781 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4782 }
4783
4784 if (NegMul && !IsScalar)
4785 Ops[0] = Builder.CreateFNeg(Ops[0]);
4786 if (NegMul && IsScalar)
4787 Ops[1] = Builder.CreateFNeg(Ops[1]);
4788 if (NegAcc)
4789 Ops[2] = Builder.CreateFNeg(Ops[2]);
4790
4791 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4792
4793 if (IsScalar)
4794 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4795 } else if (Name.starts_with("fma4.vfmadd.s")) {
4796 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4797 CI->getArgOperand(2)};
4798
4799 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4800 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4801 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4802
4803 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4804
4805 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4806 Rep, (uint64_t)0);
4807 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4808 Name.starts_with("avx512.maskz.vfmadd.s") ||
4809 Name.starts_with("avx512.mask3.vfmadd.s") ||
4810 Name.starts_with("avx512.mask3.vfmsub.s") ||
4811 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4812 bool IsMask3 = Name[11] == '3';
4813 bool IsMaskZ = Name[11] == 'z';
4814 // Drop the "avx512.mask." to make it easier.
4815 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4816 bool NegMul = Name[2] == 'n';
4817 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4818
4819 Value *A = CI->getArgOperand(0);
4820 Value *B = CI->getArgOperand(1);
4821 Value *C = CI->getArgOperand(2);
4822
4823 if (NegMul && (IsMask3 || IsMaskZ))
4824 A = Builder.CreateFNeg(A);
4825 if (NegMul && !(IsMask3 || IsMaskZ))
4826 B = Builder.CreateFNeg(B);
4827 if (NegAcc)
4828 C = Builder.CreateFNeg(C);
4829
4830 A = Builder.CreateExtractElement(A, (uint64_t)0);
4831 B = Builder.CreateExtractElement(B, (uint64_t)0);
4832 C = Builder.CreateExtractElement(C, (uint64_t)0);
4833
4834 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4835 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4836 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4837
4838 Intrinsic::ID IID;
4839 if (Name.back() == 'd')
4840 IID = Intrinsic::x86_avx512_vfmadd_f64;
4841 else
4842 IID = Intrinsic::x86_avx512_vfmadd_f32;
4843 Rep = Builder.CreateIntrinsic(IID, Ops);
4844 } else {
4845 Rep = Builder.CreateFMA(A, B, C);
4846 }
4847
4848 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4849 : IsMask3 ? C
4850 : A;
4851
4852 // For Mask3 with NegAcc, we need to create a new extractelement that
4853 // avoids the negation above.
4854 if (NegAcc && IsMask3)
4855 PassThru =
4856 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4857
4858 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4859 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4860 (uint64_t)0);
4861 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4862 Name.starts_with("avx512.mask.vfnmadd.p") ||
4863 Name.starts_with("avx512.mask.vfnmsub.p") ||
4864 Name.starts_with("avx512.mask3.vfmadd.p") ||
4865 Name.starts_with("avx512.mask3.vfmsub.p") ||
4866 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4867 Name.starts_with("avx512.maskz.vfmadd.p")) {
4868 bool IsMask3 = Name[11] == '3';
4869 bool IsMaskZ = Name[11] == 'z';
4870 // Drop the "avx512.mask." to make it easier.
4871 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4872 bool NegMul = Name[2] == 'n';
4873 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4874
4875 Value *A = CI->getArgOperand(0);
4876 Value *B = CI->getArgOperand(1);
4877 Value *C = CI->getArgOperand(2);
4878
4879 if (NegMul && (IsMask3 || IsMaskZ))
4880 A = Builder.CreateFNeg(A);
4881 if (NegMul && !(IsMask3 || IsMaskZ))
4882 B = Builder.CreateFNeg(B);
4883 if (NegAcc)
4884 C = Builder.CreateFNeg(C);
4885
4886 if (CI->arg_size() == 5 &&
4887 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4888 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4889 Intrinsic::ID IID;
4890 // Check the character before ".512" in string.
4891 if (Name[Name.size() - 5] == 's')
4892 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4893 else
4894 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4895
4896 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4897 } else {
4898 Rep = Builder.CreateFMA(A, B, C);
4899 }
4900
4901 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4902 : IsMask3 ? CI->getArgOperand(2)
4903 : CI->getArgOperand(0);
4904
4905 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4906 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4907 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4908 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4909 Intrinsic::ID IID;
4910 if (VecWidth == 128 && EltWidth == 32)
4911 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4912 else if (VecWidth == 256 && EltWidth == 32)
4913 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4914 else if (VecWidth == 128 && EltWidth == 64)
4915 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4916 else if (VecWidth == 256 && EltWidth == 64)
4917 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4918 else
4919 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4920
4921 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4922 CI->getArgOperand(2)};
4923 Ops[2] = Builder.CreateFNeg(Ops[2]);
4924 Rep = Builder.CreateIntrinsic(IID, Ops);
4925 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4926 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4927 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4928 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4929 bool IsMask3 = Name[11] == '3';
4930 bool IsMaskZ = Name[11] == 'z';
4931 // Drop the "avx512.mask." to make it easier.
4932 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4933 bool IsSubAdd = Name[3] == 's';
4934 if (CI->arg_size() == 5) {
4935 Intrinsic::ID IID;
4936 // Check the character before ".512" in string.
4937 if (Name[Name.size() - 5] == 's')
4938 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4939 else
4940 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4941
4942 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4943 CI->getArgOperand(2), CI->getArgOperand(4)};
4944 if (IsSubAdd)
4945 Ops[2] = Builder.CreateFNeg(Ops[2]);
4946
4947 Rep = Builder.CreateIntrinsic(IID, Ops);
4948 } else {
4949 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4950
4951 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4952 CI->getArgOperand(2)};
4953
4954 Function *FMA = Intrinsic::getOrInsertDeclaration(
4955 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4956 Value *Odd = Builder.CreateCall(FMA, Ops);
4957 Ops[2] = Builder.CreateFNeg(Ops[2]);
4958 Value *Even = Builder.CreateCall(FMA, Ops);
4959
4960 if (IsSubAdd)
4961 std::swap(Even, Odd);
4962
4963 SmallVector<int, 32> Idxs(NumElts);
4964 for (int i = 0; i != NumElts; ++i)
4965 Idxs[i] = i + (i % 2) * NumElts;
4966
4967 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4968 }
4969
4970 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4971 : IsMask3 ? CI->getArgOperand(2)
4972 : CI->getArgOperand(0);
4973
4974 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4975 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4976 Name.starts_with("avx512.maskz.pternlog.")) {
4977 bool ZeroMask = Name[11] == 'z';
4978 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4979 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4980 Intrinsic::ID IID;
4981 if (VecWidth == 128 && EltWidth == 32)
4982 IID = Intrinsic::x86_avx512_pternlog_d_128;
4983 else if (VecWidth == 256 && EltWidth == 32)
4984 IID = Intrinsic::x86_avx512_pternlog_d_256;
4985 else if (VecWidth == 512 && EltWidth == 32)
4986 IID = Intrinsic::x86_avx512_pternlog_d_512;
4987 else if (VecWidth == 128 && EltWidth == 64)
4988 IID = Intrinsic::x86_avx512_pternlog_q_128;
4989 else if (VecWidth == 256 && EltWidth == 64)
4990 IID = Intrinsic::x86_avx512_pternlog_q_256;
4991 else if (VecWidth == 512 && EltWidth == 64)
4992 IID = Intrinsic::x86_avx512_pternlog_q_512;
4993 else
4994 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4995
4996 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4997 CI->getArgOperand(2), CI->getArgOperand(3)};
4998 Rep = Builder.CreateIntrinsic(IID, Args);
4999 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5000 : CI->getArgOperand(0);
5001 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
5002 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
5003 Name.starts_with("avx512.maskz.vpmadd52")) {
5004 bool ZeroMask = Name[11] == 'z';
5005 bool High = Name[20] == 'h' || Name[21] == 'h';
5006 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5007 Intrinsic::ID IID;
5008 if (VecWidth == 128 && !High)
5009 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5010 else if (VecWidth == 256 && !High)
5011 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5012 else if (VecWidth == 512 && !High)
5013 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5014 else if (VecWidth == 128 && High)
5015 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5016 else if (VecWidth == 256 && High)
5017 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5018 else if (VecWidth == 512 && High)
5019 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5020 else
5021 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5022
5023 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5024 CI->getArgOperand(2)};
5025 Rep = Builder.CreateIntrinsic(IID, Args);
5026 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5027 : CI->getArgOperand(0);
5028 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5029 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5030 Name.starts_with("avx512.mask.vpermt2var.") ||
5031 Name.starts_with("avx512.maskz.vpermt2var.")) {
5032 bool ZeroMask = Name[11] == 'z';
5033 bool IndexForm = Name[17] == 'i';
5034 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5035 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5036 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5037 Name.starts_with("avx512.mask.vpdpbusds.") ||
5038 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5039 bool ZeroMask = Name[11] == 'z';
5040 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5041 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5042 Intrinsic::ID IID;
5043 if (VecWidth == 128 && !IsSaturating)
5044 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5045 else if (VecWidth == 256 && !IsSaturating)
5046 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5047 else if (VecWidth == 512 && !IsSaturating)
5048 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5049 else if (VecWidth == 128 && IsSaturating)
5050 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5051 else if (VecWidth == 256 && IsSaturating)
5052 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5053 else if (VecWidth == 512 && IsSaturating)
5054 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5055 else
5056 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5057
5058 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5059 CI->getArgOperand(2)};
5060
5061 // Input arguments types were incorrectly set to vectors of i32 before but
5062 // they should be vectors of i8. Insert bit cast when encountering the old
5063 // types
5064 if (Args[1]->getType()->isVectorTy() &&
5065 cast<VectorType>(Args[1]->getType())
5066 ->getElementType()
5067 ->isIntegerTy(32) &&
5068 Args[2]->getType()->isVectorTy() &&
5069 cast<VectorType>(Args[2]->getType())
5070 ->getElementType()
5071 ->isIntegerTy(32)) {
5072 Type *NewArgType = nullptr;
5073 if (VecWidth == 128)
5074 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5075 else if (VecWidth == 256)
5076 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5077 else if (VecWidth == 512)
5078 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5079 else
5080 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5081 CI);
5082
5083 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5084 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5085 }
5086
5087 Rep = Builder.CreateIntrinsic(IID, Args);
5088 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5089 : CI->getArgOperand(0);
5090 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5091 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5092 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5093 Name.starts_with("avx512.mask.vpdpwssds.") ||
5094 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5095 bool ZeroMask = Name[11] == 'z';
5096 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5097 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5098 Intrinsic::ID IID;
5099 if (VecWidth == 128 && !IsSaturating)
5100 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5101 else if (VecWidth == 256 && !IsSaturating)
5102 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5103 else if (VecWidth == 512 && !IsSaturating)
5104 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5105 else if (VecWidth == 128 && IsSaturating)
5106 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5107 else if (VecWidth == 256 && IsSaturating)
5108 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5109 else if (VecWidth == 512 && IsSaturating)
5110 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5111 else
5112 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5113
5114 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5115 CI->getArgOperand(2)};
5116
5117 // Input arguments types were incorrectly set to vectors of i32 before but
5118 // they should be vectors of i16. Insert bit cast when encountering the old
5119 // types
5120 if (Args[1]->getType()->isVectorTy() &&
5121 cast<VectorType>(Args[1]->getType())
5122 ->getElementType()
5123 ->isIntegerTy(32) &&
5124 Args[2]->getType()->isVectorTy() &&
5125 cast<VectorType>(Args[2]->getType())
5126 ->getElementType()
5127 ->isIntegerTy(32)) {
5128 Type *NewArgType = nullptr;
5129 if (VecWidth == 128)
5130 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5131 else if (VecWidth == 256)
5132 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5133 else if (VecWidth == 512)
5134 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5135 else
5136 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5137 CI);
5138
5139 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5140 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5141 }
5142
5143 Rep = Builder.CreateIntrinsic(IID, Args);
5144 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5145 : CI->getArgOperand(0);
5146 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5147 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5148 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5149 Name == "subborrow.u32" || Name == "subborrow.u64") {
5150 Intrinsic::ID IID;
5151 if (Name[0] == 'a' && Name.back() == '2')
5152 IID = Intrinsic::x86_addcarry_32;
5153 else if (Name[0] == 'a' && Name.back() == '4')
5154 IID = Intrinsic::x86_addcarry_64;
5155 else if (Name[0] == 's' && Name.back() == '2')
5156 IID = Intrinsic::x86_subborrow_32;
5157 else if (Name[0] == 's' && Name.back() == '4')
5158 IID = Intrinsic::x86_subborrow_64;
5159 else
5160 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5161
5162 // Make a call with 3 operands.
5163 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5164 CI->getArgOperand(2)};
5165 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5166
5167 // Extract the second result and store it.
5168 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5169 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5170 // Replace the original call result with the first result of the new call.
5171 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5172
5173 CI->replaceAllUsesWith(CF);
5174 Rep = nullptr;
5175 } else if (Name.starts_with("avx512.mask.") &&
5176 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5177 // Rep will be updated by the call in the condition.
5178 } else if (Name.starts_with("bmi.pdep.")) {
5179 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5180 } else if (Name.starts_with("bmi.pext.")) {
5181 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5182 } else
5183 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5184
5185 return Rep;
5186}
5187
5188static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5189 Function *F, IRBuilder<> &Builder) {
5190 if (Name.starts_with("neon.bfcvt")) {
5191 if (Name.starts_with("neon.bfcvtn2")) {
5192 SmallVector<int, 32> LoMask(4);
5193 std::iota(LoMask.begin(), LoMask.end(), 0);
5194 SmallVector<int, 32> ConcatMask(8);
5195 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5196 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5197 Value *Trunc =
5198 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5199 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5200 } else if (Name.starts_with("neon.bfcvtn")) {
5201 SmallVector<int, 32> ConcatMask(8);
5202 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5203 Type *V4BF16 =
5204 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5205 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5206 dbgs() << "Trunc: " << *Trunc << "\n";
5207 return Builder.CreateShuffleVector(
5208 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5209 } else {
5210 return Builder.CreateFPTrunc(CI->getOperand(0),
5211 Type::getBFloatTy(F->getContext()));
5212 }
5213 } else if (Name.starts_with("sve.fcvt")) {
5214 Intrinsic::ID NewID =
5215 StringSwitch<Intrinsic::ID>(Name)
5216 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5217 .Case("sve.fcvtnt.bf16f32",
5218 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5219 .Default(Intrinsic::not_intrinsic);
5220 if (NewID == Intrinsic::not_intrinsic)
5221 llvm_unreachable("Unhandled Intrinsic!");
5222
5223 SmallVector<Value *, 3> Args(CI->args());
5224
5225 // The original intrinsics incorrectly used a predicate based on the
5226 // smallest element type rather than the largest.
5227 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5228 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5229
5230 if (Args[1]->getType() != BadPredTy)
5231 llvm_unreachable("Unexpected predicate type!");
5232
5233 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5234 BadPredTy, Args[1]);
5235 Args[1] = Builder.CreateIntrinsic(
5236 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5237
5238 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5239 CI->getName());
5240 }
5241
5242 if (Name == "neon.vcvtfp2hf")
5243 return Builder.CreateBitCast(
5244 Builder.CreateFPTrunc(
5245 CI->getOperand(0),
5246 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5247 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5248 if (Name == "neon.vcvthf2fp")
5249 return Builder.CreateFPExt(
5250 Builder.CreateBitCast(
5251 CI->getOperand(0),
5252 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5253 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5254
5255 llvm_unreachable("Unhandled Intrinsic!");
5256}
5257
5258static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5259 IRBuilder<> &Builder) {
5260 if (Name == "mve.vctp64.old") {
5261 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5262 // correct type.
5263 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5264 CI->getArgOperand(0),
5265 /*FMFSource=*/nullptr, CI->getName());
5266 Value *C1 = Builder.CreateIntrinsic(
5267 Intrinsic::arm_mve_pred_v2i,
5268 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5269 return Builder.CreateIntrinsic(
5270 Intrinsic::arm_mve_pred_i2v,
5271 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5272 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5273 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5274 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5275 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5276 Name ==
5277 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5278 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5279 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5280 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5281 Name ==
5282 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5283 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5284 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5285 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5286 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5287 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5288 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5289 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5290 std::vector<Type *> Tys;
5291 unsigned ID = CI->getIntrinsicID();
5292 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5293 switch (ID) {
5294 case Intrinsic::arm_mve_mull_int_predicated:
5295 case Intrinsic::arm_mve_vqdmull_predicated:
5296 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5297 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5298 break;
5299 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5300 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5301 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5302 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5303 V2I1Ty};
5304 break;
5305 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5306 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5307 CI->getOperand(1)->getType(), V2I1Ty};
5308 break;
5309 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5310 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5311 CI->getOperand(2)->getType(), V2I1Ty};
5312 break;
5313 case Intrinsic::arm_cde_vcx1q_predicated:
5314 case Intrinsic::arm_cde_vcx1qa_predicated:
5315 case Intrinsic::arm_cde_vcx2q_predicated:
5316 case Intrinsic::arm_cde_vcx2qa_predicated:
5317 case Intrinsic::arm_cde_vcx3q_predicated:
5318 case Intrinsic::arm_cde_vcx3qa_predicated:
5319 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5320 break;
5321 default:
5322 llvm_unreachable("Unhandled Intrinsic!");
5323 }
5324
5325 std::vector<Value *> Ops;
5326 for (Value *Op : CI->args()) {
5327 Type *Ty = Op->getType();
5328 if (Ty->getScalarSizeInBits() == 1) {
5329 Value *C1 = Builder.CreateIntrinsic(
5330 Intrinsic::arm_mve_pred_v2i,
5331 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5332 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5333 }
5334 Ops.push_back(Op);
5335 }
5336
5337 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5338 CI->getName());
5339 }
5340 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5341}
5342
5343// These are expected to have the arguments:
5344// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5345//
5346// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5347//
5348static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5349 Function *F, IRBuilder<> &Builder) {
5350 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5351 // for compatibility.
5352 auto UpgradeLegacyWMMAIUIntrinsicCall =
5353 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5354 ArrayRef<Type *> OverloadTys) -> Value * {
5355 // Prepare arguments, append clamp=0 for compatibility
5356 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5357 Args.push_back(Builder.getFalse());
5358
5359 // Insert the declaration for the right overload types
5360 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5361 F->getParent(), F->getIntrinsicID(), OverloadTys);
5362
5363 // Copy operand bundles if any
5364 SmallVector<OperandBundleDef, 1> Bundles;
5365 CI->getOperandBundlesAsDefs(Bundles);
5366
5367 // Create the new call and copy calling properties
5368 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5369 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5370 NewCall->setCallingConv(CI->getCallingConv());
5371 NewCall->setAttributes(CI->getAttributes());
5372 NewCall->copyMetadata(*CI);
5373 return NewCall;
5374 };
5375
5376 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5377 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5378 "intrinsic should have 7 arguments");
5379 Type *T1 = CI->getArgOperand(4)->getType();
5380 Type *T2 = CI->getArgOperand(1)->getType();
5381 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5382 }
5383 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5384 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5385 "intrinsic should have 8 arguments");
5386 Type *T1 = CI->getArgOperand(4)->getType();
5387 Type *T2 = CI->getArgOperand(1)->getType();
5388 Type *T3 = CI->getArgOperand(3)->getType();
5389 Type *T4 = CI->getArgOperand(5)->getType();
5390 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5391 }
5392
5393 switch (F->getIntrinsicID()) {
5394 default:
5395 break;
5396 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5397 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5398 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5399 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5400 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5401 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5402 // Drop src0 and src1 modifiers.
5403 const Value *Op0 = CI->getArgOperand(0);
5404 const Value *Op2 = CI->getArgOperand(2);
5405 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5406 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5407 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5408 if (!ModA->isZero() || !ModB->isZero())
5409 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5410
5411 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5412 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5413 Args.push_back(CI->getArgOperand(I));
5414
5415 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5416 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5417 Overloads.push_back(Args[3]->getType());
5418 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5419 F->getParent(), F->getIntrinsicID(), Overloads);
5420
5421 SmallVector<OperandBundleDef, 1> Bundles;
5422 CI->getOperandBundlesAsDefs(Bundles);
5423
5424 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5425 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5426 NewCall->setCallingConv(CI->getCallingConv());
5427 NewCall->setAttributes(CI->getAttributes());
5428 NewCall->copyMetadata(*CI);
5429 NewCall->takeName(CI);
5430 return NewCall;
5431 }
5432 }
5433
5434 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5435 Value *LHS = CI->getArgOperand(0);
5436 Value *RHS = CI->getArgOperand(1);
5437 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5438 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5439 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5440 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5441 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5442 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5443 NewCall->setCallingConv(CI->getCallingConv());
5444 NewCall->copyMetadata(*CI);
5445 NewCall->takeName(CI);
5446 return NewCall;
5447 }
5448
5449 if (Name.starts_with("addrspacecast.nonnull")) {
5450 if (CI->getNumOperands() < 2) // Malformed bitcode.
5451 return nullptr;
5452 Value *ASC = Builder.CreateAddrSpaceCast(
5453 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5454 ASC->takeName(CI);
5455 return ASC;
5456 }
5457
5458 AtomicRMWInst::BinOp RMWOp =
5459 StringSwitch<AtomicRMWInst::BinOp>(Name)
5460 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5461 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5462 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5463 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5464 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5465 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5466 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5467 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5468 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5469 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5470 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5471 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5472 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5473
5474 unsigned NumOperands = CI->getNumOperands();
5475 if (NumOperands < 3) // Malformed bitcode.
5476 return nullptr;
5477
5478 Value *Ptr = CI->getArgOperand(0);
5479 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5480 if (!PtrTy) // Malformed.
5481 return nullptr;
5482
5483 Value *Val = CI->getArgOperand(1);
5484 if (Val->getType() != CI->getType()) // Malformed.
5485 return nullptr;
5486
5487 ConstantInt *OrderArg = nullptr;
5488 bool IsVolatile = false;
5489
5490 // These should have 5 arguments (plus the callee). A separate version of the
5491 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5492 if (NumOperands > 3)
5493 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5494
5495 // Ignore scope argument at 3
5496
5497 if (NumOperands > 5) {
5498 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5499 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5500 }
5501
5502 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5503 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5504 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5505 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5506 Order = AtomicOrdering::SequentiallyConsistent;
5507
5508 LLVMContext &Ctx = F->getContext();
5509
5510 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5511 Type *RetTy = CI->getType();
5512 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5513 if (VT->getElementType()->isIntegerTy(16)) {
5514 VectorType *AsBF16 =
5515 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5516 Val = Builder.CreateBitCast(Val, AsBF16);
5517 }
5518 }
5519
5520 // The scope argument never really worked correctly. Use agent as the most
5521 // conservative option which should still always produce the instruction.
5522 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5523 AtomicRMWInst *RMW =
5524 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5525
5526 unsigned AddrSpace = PtrTy->getAddressSpace();
5527 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5528 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5529 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5530 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5531 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5532 }
5533
5534 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5535 MDBuilder MDB(F->getContext());
5536 MDNode *RangeNotPrivate =
5537 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5538 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5539 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5540 }
5541
5542 if (IsVolatile)
5543 RMW->setVolatile(true);
5544
5545 return Builder.CreateBitCast(RMW, RetTy);
5546}
5547
5548/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5549/// plain MDNode, as it's the verifier's job to check these are the correct
5550/// types later.
5551static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5552 if (Op < CI->arg_size()) {
5553 if (MetadataAsValue *MAV =
5554 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5555 Metadata *MD = MAV->getMetadata();
5556 return dyn_cast_if_present<MDNode>(MD);
5557 }
5558 }
5559 return nullptr;
5560}
5561
5562/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5563static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5564 if (Op < CI->arg_size())
5565 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5566 return MAV->getMetadata();
5567 return nullptr;
5568}
5569
5570/// Convert debug intrinsic calls to non-instruction debug records.
5571/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5572/// \p CI - The debug intrinsic call.
5573static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5574 DbgRecord *DR = nullptr;
5575 if (Name == "label") {
5576 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5577 } else if (Name == "assign") {
5578 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5579 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5580 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5581 unwrapMAVMetadataOp(CI, 4),
5582 /*The address is a Value ref, it will be stored as a Metadata */
5583 unwrapMAVOp(CI, 5));
5584 } else if (Name == "declare") {
5585 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5586 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5587 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5588 } else if (Name == "addr") {
5589 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5590 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5591 // Don't try to add something to the expression if it's not an expression.
5592 // Instead, allow the verifier to fail later.
5593 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5594 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5595 }
5596 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5597 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5598 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5599 } else if (Name == "value") {
5600 // An old version of dbg.value had an extra offset argument.
5601 unsigned VarOp = 1;
5602 unsigned ExprOp = 2;
5603 if (CI->arg_size() == 4) {
5604 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5605 // Nonzero offset dbg.values get dropped without a replacement.
5606 if (!Offset || !Offset->isNullValue())
5607 return;
5608 VarOp = 2;
5609 ExprOp = 3;
5610 }
5611 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5612 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5613 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5614 nullptr);
5615 }
5616 DR->setDebugLoc(CI->getDebugLoc());
5617 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5618 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5619}
5620
5621static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5622 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5623 if (!Offset)
5624 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5625 int64_t OffsetVal = Offset->getSExtValue();
5626 return Builder.CreateIntrinsic(OffsetVal >= 0
5627 ? Intrinsic::vector_splice_left
5628 : Intrinsic::vector_splice_right,
5629 CI->getType(),
5630 {CI->getArgOperand(0), CI->getArgOperand(1),
5631 Builder.getInt32(std::abs(OffsetVal))});
5632}
5633
5634static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5635 Function *F, IRBuilder<> &Builder) {
5636 if (Name.starts_with("to.fp16")) {
5637 Value *Cast =
5638 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5639 return Builder.CreateBitCast(Cast, CI->getType());
5640 }
5641
5642 if (Name.starts_with("from.fp16")) {
5643 Value *Cast =
5644 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5645 return Builder.CreateFPExt(Cast, CI->getType());
5646 }
5647
5648 return nullptr;
5649}
5650
5651static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5652 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5653 if (!MD || !isa<MDString>(MD))
5654 return ICmpInst::BAD_ICMP_PREDICATE;
5655 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5656 .Case("eq", ICmpInst::ICMP_EQ)
5657 .Case("ne", ICmpInst::ICMP_NE)
5658 .Case("ugt", ICmpInst::ICMP_UGT)
5659 .Case("uge", ICmpInst::ICMP_UGE)
5660 .Case("ult", ICmpInst::ICMP_ULT)
5661 .Case("ule", ICmpInst::ICMP_ULE)
5662 .Case("sgt", ICmpInst::ICMP_SGT)
5663 .Case("sge", ICmpInst::ICMP_SGE)
5664 .Case("slt", ICmpInst::ICMP_SLT)
5665 .Case("sle", ICmpInst::ICMP_SLE)
5666 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5667}
5668
5669static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5670 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5671 if (!MD || !isa<MDString>(MD))
5672 return FCmpInst::BAD_FCMP_PREDICATE;
5673 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5674 .Case("oeq", FCmpInst::FCMP_OEQ)
5675 .Case("ogt", FCmpInst::FCMP_OGT)
5676 .Case("oge", FCmpInst::FCMP_OGE)
5677 .Case("olt", FCmpInst::FCMP_OLT)
5678 .Case("ole", FCmpInst::FCMP_OLE)
5679 .Case("one", FCmpInst::FCMP_ONE)
5680 .Case("ord", FCmpInst::FCMP_ORD)
5681 .Case("uno", FCmpInst::FCMP_UNO)
5682 .Case("ueq", FCmpInst::FCMP_UEQ)
5683 .Case("ugt", FCmpInst::FCMP_UGT)
5684 .Case("uge", FCmpInst::FCMP_UGE)
5685 .Case("ult", FCmpInst::FCMP_ULT)
5686 .Case("ule", FCmpInst::FCMP_ULE)
5687 .Case("une", FCmpInst::FCMP_UNE)
5688 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5689}
5690
5691static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5692 IRBuilder<> &Builder) {
5693 Value *Rep;
5694 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5695 if (Opcode && Instruction::isUnaryOp(Opcode))
5696 Rep =
5697 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5698 else if (Opcode && Instruction::isBinaryOp(Opcode))
5699 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5700 CI->getArgOperand(0), CI->getArgOperand(1));
5701 else if (Opcode && Instruction::isCast(Opcode))
5702 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5703 CI->getType());
5704 else if (Opcode == Instruction::ICmp)
5705 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5706 CI->getArgOperand(0), CI->getArgOperand(1));
5707 else if (Opcode == Instruction::FCmp)
5708 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5709 CI->getArgOperand(0), CI->getArgOperand(1));
5710 else if (Opcode == Instruction::Select)
5711 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5712 CI->getArgOperand(2));
5713 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5714 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5715 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5716 } else
5717 llvm_unreachable("Unexpected vp intrinsic");
5718 Rep->takeName(CI);
5719 return Rep;
5720}
5721
5722static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5723 IRBuilder<> &Builder) {
5724 Intrinsic::ID IID = NewFn->getIntrinsicID();
5725
5726 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5727 if (Defaults.empty())
5728 return false;
5729
5730 unsigned OldArgCount = CI->arg_size();
5731 unsigned NewArgCount = NewFn->arg_size();
5732
5733 if (OldArgCount < FirstDefault)
5734 return false;
5735
5736 // More arguments than the new intrinsic accepts, cannot upgrade.
5737 if (OldArgCount > NewArgCount)
5738 return false;
5739
5740 // The call already passes the defaulted arguments explicitly; only the
5741 // callee is still the old, shorter declaration, so retarget it.
5742 if (OldArgCount == NewArgCount) {
5743 if (CI->getFunctionType() != NewFn->getFunctionType())
5744 return false;
5745 CI->setCalledFunction(NewFn);
5746 return true;
5747 }
5748
5749 // OldArgCount < NewArgCount: Fill in each missing trailing default
5750 // argument from the table.
5751 SmallVector<Value *, 8> NewArgs(CI->args());
5752
5753 FunctionType *NewFT = NewFn->getFunctionType();
5754 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5755 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5756 "missing argument outside the default range");
5757 Type *ParamTy = NewFT->getParamType(Idx);
5758
5759 // Only integer types are supported (i1, i8, i16, i32, i64).
5760 if (!ParamTy->isIntegerTy())
5761 return false;
5762 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5763 }
5764
5765 // Preserve operand bundles by creating the call with them.
5766 SmallVector<OperandBundleDef, 1> OpBundles;
5767 CI->getOperandBundlesAsDefs(OpBundles);
5768 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5769
5770 NewCall->takeName(CI);
5771 NewCall->setCallingConv(CI->getCallingConv());
5772 NewCall->copyMetadata(*CI);
5773 if (auto *OldCI = dyn_cast<CallInst>(CI))
5774 NewCall->setTailCallKind(OldCI->getTailCallKind());
5775
5776 CI->replaceAllUsesWith(NewCall);
5777 CI->eraseFromParent();
5778 return true;
5779}
5780
5781/// Upgrade a call to an old intrinsic. All argument and return casting must be
5782/// provided to seamlessly integrate with existing context.
5783void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5784 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5785 // checks the callee's function type matches. It's likely we need to handle
5786 // type changes here.
5787 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5788 if (!F)
5789 return;
5790
5791 LLVMContext &C = CI->getContext();
5792 IRBuilder<> Builder(CI->getIterator());
5793 if (isa<FPMathOperator>(CI))
5794 Builder.setFastMathFlags(CI->getFastMathFlags());
5795
5796 if (!NewFn) {
5797 // Get the Function's name.
5798 StringRef Name = F->getName();
5799 if (!Name.consume_front("llvm."))
5800 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5801
5802 bool IsX86 = Name.consume_front("x86.");
5803 bool IsNVVM = Name.consume_front("nvvm.");
5804 bool IsAArch64 = Name.consume_front("aarch64.");
5805 bool IsARM = Name.consume_front("arm.");
5806 bool IsAMDGCN = Name.consume_front("amdgcn.");
5807 bool IsDbg = Name.consume_front("dbg.");
5808 bool IsOldSplice =
5809 (Name.consume_front("experimental.vector.splice") ||
5810 Name.consume_front("vector.splice")) &&
5811 !(Name.starts_with(".left") || Name.starts_with(".right"));
5812 Value *Rep = nullptr;
5813
5814 if (!IsX86 && Name == "stackprotectorcheck") {
5815 Rep = nullptr;
5816 } else if (IsNVVM) {
5817 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5818 } else if (IsX86) {
5819 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5820 } else if (IsAArch64) {
5821 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5822 } else if (IsARM) {
5823 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5824 } else if (IsAMDGCN) {
5825 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5826 } else if (IsDbg) {
5827 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5828 } else if (IsOldSplice) {
5829 Rep = upgradeVectorSplice(CI, Builder);
5830 } else if (Name.consume_front("convert.")) {
5831 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5832 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5833 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5834 Rep = nullptr;
5835 } else if (shouldUpgradeVPIntrinsic(Name)) {
5836 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5837 } else {
5838 llvm_unreachable("Unknown function for CallBase upgrade.");
5839 }
5840
5841 if (Rep)
5842 CI->replaceAllUsesWith(Rep);
5843 CI->eraseFromParent();
5844 return;
5845 }
5846
5847 const auto &DefaultCase = [&]() -> void {
5848 if (F == NewFn)
5849 return;
5850
5851 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5852 // Handle generic mangling change.
5853 assert(
5854 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5855 "Unknown function for CallBase upgrade and isn't just a name change");
5856 CI->setCalledFunction(NewFn);
5857 return;
5858 }
5859
5860 // This must be an upgrade from a named to a literal struct.
5861 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5862 assert(OldST != NewFn->getReturnType() &&
5863 "Return type must have changed");
5864 assert(OldST->getNumElements() ==
5865 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5866 "Must have same number of elements");
5867
5868 SmallVector<Value *> Args(CI->args());
5869 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5870 NewCI->setAttributes(CI->getAttributes());
5871 Value *Res = PoisonValue::get(OldST);
5872 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5873 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5874 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5875 }
5876 CI->replaceAllUsesWith(Res);
5877 CI->eraseFromParent();
5878 return;
5879 }
5880
5881 // We're probably about to produce something invalid. Let the verifier catch
5882 // it instead of dying here.
5883 CI->setCalledOperand(
5884 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5885 return;
5886 };
5887 CallInst *NewCall = nullptr;
5888 switch (NewFn->getIntrinsicID()) {
5889 default: {
5890 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5891 return;
5892 DefaultCase();
5893 return;
5894 }
5895 case Intrinsic::arm_neon_vst1:
5896 case Intrinsic::arm_neon_vst2:
5897 case Intrinsic::arm_neon_vst3:
5898 case Intrinsic::arm_neon_vst4:
5899 case Intrinsic::arm_neon_vst2lane:
5900 case Intrinsic::arm_neon_vst3lane:
5901 case Intrinsic::arm_neon_vst4lane: {
5902 SmallVector<Value *, 4> Args(CI->args());
5903 NewCall = Builder.CreateCall(NewFn, Args);
5904 break;
5905 }
5906 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5907 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5908 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5909 LLVMContext &Ctx = F->getParent()->getContext();
5910 SmallVector<Value *, 4> Args(CI->args());
5911 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5912 cast<ConstantInt>(Args[3])->getZExtValue());
5913 NewCall = Builder.CreateCall(NewFn, Args);
5914 break;
5915 }
5916 case Intrinsic::aarch64_sve_ld3_sret:
5917 case Intrinsic::aarch64_sve_ld4_sret:
5918 case Intrinsic::aarch64_sve_ld2_sret: {
5919 // Is this a trivial remangle of the name to support ptr address spaces?
5920 if (isa<StructType>(F->getReturnType())) {
5921 DefaultCase();
5922 return;
5923 }
5924
5925 StringRef Name = F->getName();
5926 Name = Name.substr(5);
5927 unsigned N = StringSwitch<unsigned>(Name)
5928 .StartsWith("aarch64.sve.ld2", 2)
5929 .StartsWith("aarch64.sve.ld3", 3)
5930 .StartsWith("aarch64.sve.ld4", 4)
5931 .Default(0);
5932 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5933 unsigned MinElts = RetTy->getMinNumElements() / N;
5934 SmallVector<Value *, 2> Args(CI->args());
5935 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5936 Value *Ret = llvm::PoisonValue::get(RetTy);
5937 for (unsigned I = 0; I < N; I++) {
5938 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5939 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5940 }
5941 NewCall = dyn_cast<CallInst>(Ret);
5942 break;
5943 }
5944
5945 case Intrinsic::coro_end_async:
5946 case Intrinsic::coro_end: {
5947 SmallVector<Value *, 3> Args(CI->args());
5948 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5949 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5950 NewCall = Builder.CreateCall(NewFn, Args);
5951
5952 if (!CI->getType()->isVoidTy()) {
5953 if (!CI->use_empty()) {
5954 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5955 CI->getModule(), Intrinsic::coro_is_in_ramp);
5956 Value *InRamp = Builder.CreateCall(IsInRamp);
5957 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5958 }
5959 CI->eraseFromParent();
5960 return;
5961 }
5962
5963 break;
5964 }
5965
5966 case Intrinsic::vector_extract: {
5967 StringRef Name = F->getName();
5968 Name = Name.substr(5); // Strip llvm
5969 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5970 DefaultCase();
5971 return;
5972 }
5973 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5974 unsigned MinElts = RetTy->getMinNumElements();
5975 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5976 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5977 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5978 break;
5979 }
5980
5981 case Intrinsic::vector_insert: {
5982 StringRef Name = F->getName();
5983 Name = Name.substr(5);
5984 if (!Name.starts_with("aarch64.sve.tuple")) {
5985 DefaultCase();
5986 return;
5987 }
5988 if (Name.starts_with("aarch64.sve.tuple.set")) {
5989 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5990 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5991 Value *NewIdx =
5992 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5993 NewCall = Builder.CreateCall(
5994 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5995 break;
5996 }
5997 if (Name.starts_with("aarch64.sve.tuple.create")) {
5998 unsigned N = StringSwitch<unsigned>(Name)
5999 .StartsWith("aarch64.sve.tuple.create2", 2)
6000 .StartsWith("aarch64.sve.tuple.create3", 3)
6001 .StartsWith("aarch64.sve.tuple.create4", 4)
6002 .Default(0);
6003 assert(N > 1 && "Create is expected to be between 2-4");
6004 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6005 Value *Ret = llvm::PoisonValue::get(RetTy);
6006 unsigned MinElts = RetTy->getMinNumElements() / N;
6007 for (unsigned I = 0; I < N; I++) {
6008 Value *V = CI->getArgOperand(I);
6009 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
6010 }
6011 NewCall = dyn_cast<CallInst>(Ret);
6012 }
6013 break;
6014 }
6015
6016 case Intrinsic::arm_neon_bfdot:
6017 case Intrinsic::arm_neon_bfmmla:
6018 case Intrinsic::arm_neon_bfmlalb:
6019 case Intrinsic::arm_neon_bfmlalt:
6020 case Intrinsic::aarch64_neon_bfdot:
6021 case Intrinsic::aarch64_neon_bfmmla:
6022 case Intrinsic::aarch64_neon_bfmlalb:
6023 case Intrinsic::aarch64_neon_bfmlalt: {
6024 SmallVector<Value *, 3> Args;
6025 assert(CI->arg_size() == 3 &&
6026 "Mismatch between function args and call args");
6027 size_t OperandWidth =
6028 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
6029 assert((OperandWidth == 64 || OperandWidth == 128) &&
6030 "Unexpected operand width");
6031 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6032 auto Iter = CI->args().begin();
6033 Args.push_back(*Iter++);
6034 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6035 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6036 NewCall = Builder.CreateCall(NewFn, Args);
6037 break;
6038 }
6039
6040 case Intrinsic::bitreverse:
6041 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6042 break;
6043
6044 case Intrinsic::ctlz:
6045 case Intrinsic::cttz: {
6046 if (CI->arg_size() != 1) {
6047 DefaultCase();
6048 return;
6049 }
6050
6051 NewCall =
6052 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6053 break;
6054 }
6055
6056 case Intrinsic::objectsize: {
6057 Value *NullIsUnknownSize =
6058 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6059 Value *Dynamic =
6060 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6061 NewCall = Builder.CreateCall(
6062 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6063 break;
6064 }
6065
6066 case Intrinsic::ctpop:
6067 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6068 break;
6069 case Intrinsic::dbg_value: {
6070 StringRef Name = F->getName();
6071 Name = Name.substr(5); // Strip llvm.
6072 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6073 if (Name.starts_with("dbg.addr")) {
6074 DIExpression *Expr = cast<DIExpression>(
6075 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6076 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6077 NewCall =
6078 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6079 MetadataAsValue::get(C, Expr)});
6080 break;
6081 }
6082
6083 // Upgrade from the old version that had an extra offset argument.
6084 assert(CI->arg_size() == 4);
6085 // Drop nonzero offsets instead of attempting to upgrade them.
6086 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6087 if (Offset->isNullValue()) {
6088 NewCall = Builder.CreateCall(
6089 NewFn,
6090 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6091 break;
6092 }
6093 CI->eraseFromParent();
6094 return;
6095 }
6096
6097 case Intrinsic::ptr_annotation:
6098 // Upgrade from versions that lacked the annotation attribute argument.
6099 if (CI->arg_size() != 4) {
6100 DefaultCase();
6101 return;
6102 }
6103
6104 // Create a new call with an added null annotation attribute argument.
6105 NewCall = Builder.CreateCall(
6106 NewFn,
6107 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6108 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6109 NewCall->takeName(CI);
6110 CI->replaceAllUsesWith(NewCall);
6111 CI->eraseFromParent();
6112 return;
6113
6114 case Intrinsic::var_annotation:
6115 // Upgrade from versions that lacked the annotation attribute argument.
6116 if (CI->arg_size() != 4) {
6117 DefaultCase();
6118 return;
6119 }
6120 // Create a new call with an added null annotation attribute argument.
6121 NewCall = Builder.CreateCall(
6122 NewFn,
6123 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6124 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6125 NewCall->takeName(CI);
6126 CI->replaceAllUsesWith(NewCall);
6127 CI->eraseFromParent();
6128 return;
6129
6130 case Intrinsic::riscv_aes32dsi:
6131 case Intrinsic::riscv_aes32dsmi:
6132 case Intrinsic::riscv_aes32esi:
6133 case Intrinsic::riscv_aes32esmi:
6134 case Intrinsic::riscv_sm4ks:
6135 case Intrinsic::riscv_sm4ed: {
6136 // The last argument to these intrinsics used to be i8 and changed to i32.
6137 // The type overload for sm4ks and sm4ed was removed.
6138 Value *Arg2 = CI->getArgOperand(2);
6139 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6140 return;
6141
6142 Value *Arg0 = CI->getArgOperand(0);
6143 Value *Arg1 = CI->getArgOperand(1);
6144 if (CI->getType()->isIntegerTy(64)) {
6145 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6146 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6147 }
6148
6149 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6150 cast<ConstantInt>(Arg2)->getZExtValue());
6151
6152 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6153 Value *Res = NewCall;
6154 if (Res->getType() != CI->getType())
6155 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6156 NewCall->takeName(CI);
6157 CI->replaceAllUsesWith(Res);
6158 CI->eraseFromParent();
6159 return;
6160 }
6161 case Intrinsic::nvvm_mapa_shared_cluster: {
6162 // Create a new call with the correct address space.
6163 NewCall =
6164 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6165 Value *Res = NewCall;
6166 Res = Builder.CreateAddrSpaceCast(
6167 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6168 NewCall->takeName(CI);
6169 CI->replaceAllUsesWith(Res);
6170 CI->eraseFromParent();
6171 return;
6172 }
6173 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6174 SmallVector<Value *, 4> Args(CI->args());
6175 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6176 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6177 Args[0] = Builder.CreateAddrSpaceCast(
6178 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6179
6180 // Append the missing trailing flag_valid_pattern (0 = disabled).
6181 Args.push_back(Builder.getInt32(0));
6182
6183 NewCall = Builder.CreateCall(NewFn, Args);
6184 NewCall->takeName(CI);
6185 CI->replaceAllUsesWith(NewCall);
6186 CI->eraseFromParent();
6187 return;
6188 }
6189 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6190 // (dst, mbar, src, size, ch, flag_ch)
6191 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6192 // i32 0 /* flag_valid_pattern=disabled */)
6193 SmallVector<Value *, 10> Args;
6194 for (unsigned I = 0; I < 4; ++I)
6195 Args.push_back(CI->getArgOperand(I));
6196 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6197 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6198 Args.push_back(CI->getArgOperand(4)); // cache_hint
6199 Args.push_back(CI->getArgOperand(5)); // flag_ch
6200 Args.push_back(Builder.getInt1(false)); // flag_oob
6201 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6202
6203 NewCall = Builder.CreateCall(NewFn, Args);
6204 NewCall->takeName(CI);
6205 CI->replaceAllUsesWith(NewCall);
6206 CI->eraseFromParent();
6207 return;
6208 }
6209 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6210 // Create a new call with the correct address space.
6211 SmallVector<Value *, 4> Args(CI->args());
6212 Args[0] = Builder.CreateAddrSpaceCast(
6213 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6214
6215 NewCall = Builder.CreateCall(NewFn, Args);
6216 NewCall->takeName(CI);
6217 CI->replaceAllUsesWith(NewCall);
6218 CI->eraseFromParent();
6219 return;
6220 }
6221 // clang-format off
6222#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6223 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6224 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6225#undef G2S_CLUSTER_CASE
6226 {
6227 SmallVector<Value *, 16> Args(CI->args());
6228 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6229 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6230 Args[0] = Builder.CreateAddrSpaceCast(
6231 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6232
6233 // Append the missing trailing arguments with default values (cta_group,
6234 // flag_valid_pattern).
6235 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6236 Args.push_back(Builder.getInt32(0));
6237
6238 NewCall = Builder.CreateCall(NewFn, Args);
6239 NewCall->takeName(CI);
6240 CI->replaceAllUsesWith(NewCall);
6241 CI->eraseFromParent();
6242 return;
6243 }
6244
6245#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6246 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6247 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6248#undef G2S_CTA_CASE
6249 {
6250 SmallVector<Value *, 16> Args(CI->args());
6251 // Append the missing trailing flag_valid_pattern argument with default
6252 // value 0.
6253 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6254 "expected only the trailing flag_valid_pattern to be missing");
6255 Args.push_back(Builder.getInt32(0));
6256
6257 NewCall = Builder.CreateCall(NewFn, Args);
6258 NewCall->takeName(CI);
6259 CI->replaceAllUsesWith(NewCall);
6260 CI->eraseFromParent();
6261 return;
6262 }
6263#undef NVVM_TMA_G2S_MODES
6264 // clang-format on
6265
6266 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6267 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6268 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6269 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6270 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6271 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6272 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6273 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6274 StringRef Name = F->getName();
6275 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6276 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6277
6278 SmallVector<Value *, 16> Args(CI->args());
6279 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6280 NewCall = Builder.CreateCall(NewFn, Args);
6281 break;
6282 }
6283 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6284 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6285 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6286 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6287 NewCall =
6288 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6289 Builder.getFalse()});
6290 break;
6291 case Intrinsic::nvvm_mbarrier_init: {
6292 SmallVector<Value *, 3> Args(CI->args());
6293 // The .shared variant folded into the overloaded form without gaining an
6294 // operand, so only the pre-layout two-argument form needs one appended.
6295 if (Args.size() == 2)
6296 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6297 NewCall = Builder.CreateCall(NewFn, Args);
6298 break;
6299 }
6300 case Intrinsic::riscv_sha256sig0:
6301 case Intrinsic::riscv_sha256sig1:
6302 case Intrinsic::riscv_sha256sum0:
6303 case Intrinsic::riscv_sha256sum1:
6304 case Intrinsic::riscv_sm3p0:
6305 case Intrinsic::riscv_sm3p1: {
6306 // The last argument to these intrinsics used to be i8 and changed to i32.
6307 // The type overload for sm4ks and sm4ed was removed.
6308 if (!CI->getType()->isIntegerTy(64))
6309 return;
6310
6311 Value *Arg =
6312 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6313
6314 NewCall = Builder.CreateCall(NewFn, Arg);
6315 Value *Res =
6316 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6317 NewCall->takeName(CI);
6318 CI->replaceAllUsesWith(Res);
6319 CI->eraseFromParent();
6320 return;
6321 }
6322
6323 case Intrinsic::x86_xop_vfrcz_ss:
6324 case Intrinsic::x86_xop_vfrcz_sd:
6325 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6326 break;
6327
6328 case Intrinsic::x86_xop_vpermil2pd:
6329 case Intrinsic::x86_xop_vpermil2ps:
6330 case Intrinsic::x86_xop_vpermil2pd_256:
6331 case Intrinsic::x86_xop_vpermil2ps_256: {
6332 SmallVector<Value *, 4> Args(CI->args());
6333 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6334 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6335 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6336 NewCall = Builder.CreateCall(NewFn, Args);
6337 break;
6338 }
6339
6340 case Intrinsic::x86_sse41_ptestc:
6341 case Intrinsic::x86_sse41_ptestz:
6342 case Intrinsic::x86_sse41_ptestnzc: {
6343 // The arguments for these intrinsics used to be v4f32, and changed
6344 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6345 // So, the only thing required is a bitcast for both arguments.
6346 // First, check the arguments have the old type.
6347 Value *Arg0 = CI->getArgOperand(0);
6348 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6349 return;
6350
6351 // Old intrinsic, add bitcasts
6352 Value *Arg1 = CI->getArgOperand(1);
6353
6354 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6355
6356 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6357 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6358
6359 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6360 break;
6361 }
6362
6363 case Intrinsic::x86_rdtscp: {
6364 // This used to take 1 arguments. If we have no arguments, it is already
6365 // upgraded.
6366 if (CI->getNumOperands() == 0)
6367 return;
6368
6369 NewCall = Builder.CreateCall(NewFn);
6370 // Extract the second result and store it.
6371 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6372 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6373 // Replace the original call result with the first result of the new call.
6374 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6375
6376 NewCall->takeName(CI);
6377 CI->replaceAllUsesWith(TSC);
6378 CI->eraseFromParent();
6379 return;
6380 }
6381
6382 case Intrinsic::x86_sse41_insertps:
6383 case Intrinsic::x86_sse41_dppd:
6384 case Intrinsic::x86_sse41_dpps:
6385 case Intrinsic::x86_sse41_mpsadbw:
6386 case Intrinsic::x86_avx_dp_ps_256:
6387 case Intrinsic::x86_avx2_mpsadbw: {
6388 // Need to truncate the last argument from i32 to i8 -- this argument models
6389 // an inherently 8-bit immediate operand to these x86 instructions.
6390 SmallVector<Value *, 4> Args(CI->args());
6391
6392 // Replace the last argument with a trunc.
6393 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6394 NewCall = Builder.CreateCall(NewFn, Args);
6395 break;
6396 }
6397
6398 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6399 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6400 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6401 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6402 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6403 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6404 SmallVector<Value *, 4> Args(CI->args());
6405 unsigned NumElts =
6406 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6407 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6408
6409 NewCall = Builder.CreateCall(NewFn, Args);
6410 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6411
6412 NewCall->takeName(CI);
6413 CI->replaceAllUsesWith(Res);
6414 CI->eraseFromParent();
6415 return;
6416 }
6417
6418 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6419 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6420 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6421 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6422 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6423 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6424 SmallVector<Value *, 4> Args(CI->args());
6425 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6426 if (NewFn->getIntrinsicID() ==
6427 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6428 Args[1] = Builder.CreateBitCast(
6429 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6430
6431 NewCall = Builder.CreateCall(NewFn, Args);
6432 Value *Res = Builder.CreateBitCast(
6433 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6434
6435 NewCall->takeName(CI);
6436 CI->replaceAllUsesWith(Res);
6437 CI->eraseFromParent();
6438 return;
6439 }
6440 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6441 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6442 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6443 SmallVector<Value *, 4> Args(CI->args());
6444 unsigned NumElts =
6445 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6446 Args[1] = Builder.CreateBitCast(
6447 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6448 Args[2] = Builder.CreateBitCast(
6449 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6450
6451 NewCall = Builder.CreateCall(NewFn, Args);
6452 break;
6453 }
6454
6455 case Intrinsic::thread_pointer: {
6456 NewCall = Builder.CreateCall(NewFn, {});
6457 break;
6458 }
6459
6460 case Intrinsic::memcpy:
6461 case Intrinsic::memmove:
6462 case Intrinsic::memset: {
6463 // We have to make sure that the call signature is what we're expecting.
6464 // We only want to change the old signatures by removing the alignment arg:
6465 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6466 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6467 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6468 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6469 // Note: i8*'s in the above can be any pointer type
6470 if (CI->arg_size() != 5) {
6471 DefaultCase();
6472 return;
6473 }
6474 // Remove alignment argument (3), and add alignment attributes to the
6475 // dest/src pointers.
6476 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6477 CI->getArgOperand(2), CI->getArgOperand(4)};
6478 NewCall = Builder.CreateCall(NewFn, Args);
6479 AttributeList OldAttrs = CI->getAttributes();
6480 AttributeList NewAttrs = AttributeList::get(
6481 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6482 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6483 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6484 NewCall->setAttributes(NewAttrs);
6485 auto *MemCI = cast<MemIntrinsic>(NewCall);
6486 // All mem intrinsics support dest alignment.
6487 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6488 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6489 // Memcpy/Memmove also support source alignment.
6490 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6491 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6492 break;
6493 }
6494
6495 case Intrinsic::masked_load:
6496 case Intrinsic::masked_gather:
6497 case Intrinsic::masked_store:
6498 case Intrinsic::masked_scatter: {
6499 if (CI->arg_size() != 4) {
6500 DefaultCase();
6501 return;
6502 }
6503
6504 auto GetMaybeAlign = [](Value *Op) {
6505 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6506 uint64_t Val = CI->getZExtValue();
6507 if (Val == 0)
6508 return MaybeAlign();
6509 if (isPowerOf2_64(Val))
6510 return MaybeAlign(Val);
6511 }
6512 reportFatalUsageError("Invalid alignment argument");
6513 };
6514 auto GetAlign = [&](Value *Op) {
6515 MaybeAlign Align = GetMaybeAlign(Op);
6516 if (Align)
6517 return *Align;
6518 reportFatalUsageError("Invalid zero alignment argument");
6519 };
6520
6521 const DataLayout &DL = CI->getDataLayout();
6522 switch (NewFn->getIntrinsicID()) {
6523 case Intrinsic::masked_load:
6524 NewCall = Builder.CreateMaskedLoad(
6525 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6526 CI->getArgOperand(2), CI->getArgOperand(3));
6527 break;
6528 case Intrinsic::masked_gather:
6529 NewCall = Builder.CreateMaskedGather(
6530 CI->getType(), CI->getArgOperand(0),
6531 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6532 CI->getType()->getScalarType()),
6533 CI->getArgOperand(2), CI->getArgOperand(3));
6534 break;
6535 case Intrinsic::masked_store:
6536 NewCall = Builder.CreateMaskedStore(
6537 CI->getArgOperand(0), CI->getArgOperand(1),
6538 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6539 break;
6540 case Intrinsic::masked_scatter:
6541 NewCall = Builder.CreateMaskedScatter(
6542 CI->getArgOperand(0), CI->getArgOperand(1),
6543 DL.getValueOrABITypeAlignment(
6544 GetMaybeAlign(CI->getArgOperand(2)),
6545 CI->getArgOperand(0)->getType()->getScalarType()),
6546 CI->getArgOperand(3));
6547 break;
6548 default:
6549 llvm_unreachable("Unexpected intrinsic ID");
6550 }
6551 // Previous metadata is still valid.
6552 NewCall->copyMetadata(*CI);
6553 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6554 break;
6555 }
6556
6557 case Intrinsic::lifetime_start:
6558 case Intrinsic::lifetime_end: {
6559 if (CI->arg_size() != 2) {
6560 DefaultCase();
6561 return;
6562 }
6563
6564 Value *Ptr = CI->getArgOperand(1);
6565 // Try to strip pointer casts, such that the lifetime works on an alloca.
6566 Ptr = Ptr->stripPointerCasts();
6567 if (isa<AllocaInst>(Ptr)) {
6568 // Don't use NewFn, as we might have looked through an addrspacecast.
6569 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6570 NewCall = Builder.CreateLifetimeStart(Ptr);
6571 else
6572 NewCall = Builder.CreateLifetimeEnd(Ptr);
6573 break;
6574 }
6575
6576 // Otherwise remove the lifetime marker.
6577 CI->eraseFromParent();
6578 return;
6579 }
6580
6581 case Intrinsic::x86_avx512_vpdpbusd_128:
6582 case Intrinsic::x86_avx512_vpdpbusd_256:
6583 case Intrinsic::x86_avx512_vpdpbusd_512:
6584 case Intrinsic::x86_avx512_vpdpbusds_128:
6585 case Intrinsic::x86_avx512_vpdpbusds_256:
6586 case Intrinsic::x86_avx512_vpdpbusds_512:
6587 case Intrinsic::x86_avx2_vpdpbssd_128:
6588 case Intrinsic::x86_avx2_vpdpbssd_256:
6589 case Intrinsic::x86_avx10_vpdpbssd_512:
6590 case Intrinsic::x86_avx2_vpdpbssds_128:
6591 case Intrinsic::x86_avx2_vpdpbssds_256:
6592 case Intrinsic::x86_avx10_vpdpbssds_512:
6593 case Intrinsic::x86_avx2_vpdpbsud_128:
6594 case Intrinsic::x86_avx2_vpdpbsud_256:
6595 case Intrinsic::x86_avx10_vpdpbsud_512:
6596 case Intrinsic::x86_avx2_vpdpbsuds_128:
6597 case Intrinsic::x86_avx2_vpdpbsuds_256:
6598 case Intrinsic::x86_avx10_vpdpbsuds_512:
6599 case Intrinsic::x86_avx2_vpdpbuud_128:
6600 case Intrinsic::x86_avx2_vpdpbuud_256:
6601 case Intrinsic::x86_avx10_vpdpbuud_512:
6602 case Intrinsic::x86_avx2_vpdpbuuds_128:
6603 case Intrinsic::x86_avx2_vpdpbuuds_256:
6604 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6605 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6606 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6607 CI->getArgOperand(2)};
6608 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6609 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6610 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6611
6612 NewCall = Builder.CreateCall(NewFn, Args);
6613 break;
6614 }
6615 case Intrinsic::x86_avx512_vpdpwssd_128:
6616 case Intrinsic::x86_avx512_vpdpwssd_256:
6617 case Intrinsic::x86_avx512_vpdpwssd_512:
6618 case Intrinsic::x86_avx512_vpdpwssds_128:
6619 case Intrinsic::x86_avx512_vpdpwssds_256:
6620 case Intrinsic::x86_avx512_vpdpwssds_512:
6621 case Intrinsic::x86_avx2_vpdpwsud_128:
6622 case Intrinsic::x86_avx2_vpdpwsud_256:
6623 case Intrinsic::x86_avx10_vpdpwsud_512:
6624 case Intrinsic::x86_avx2_vpdpwsuds_128:
6625 case Intrinsic::x86_avx2_vpdpwsuds_256:
6626 case Intrinsic::x86_avx10_vpdpwsuds_512:
6627 case Intrinsic::x86_avx2_vpdpwusd_128:
6628 case Intrinsic::x86_avx2_vpdpwusd_256:
6629 case Intrinsic::x86_avx10_vpdpwusd_512:
6630 case Intrinsic::x86_avx2_vpdpwusds_128:
6631 case Intrinsic::x86_avx2_vpdpwusds_256:
6632 case Intrinsic::x86_avx10_vpdpwusds_512:
6633 case Intrinsic::x86_avx2_vpdpwuud_128:
6634 case Intrinsic::x86_avx2_vpdpwuud_256:
6635 case Intrinsic::x86_avx10_vpdpwuud_512:
6636 case Intrinsic::x86_avx2_vpdpwuuds_128:
6637 case Intrinsic::x86_avx2_vpdpwuuds_256:
6638 case Intrinsic::x86_avx10_vpdpwuuds_512:
6639 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6640 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6641 CI->getArgOperand(2)};
6642 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6643 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6644 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6645
6646 NewCall = Builder.CreateCall(NewFn, Args);
6647 break;
6648 }
6649 assert(NewCall && "Should have either set this variable or returned through "
6650 "the default case");
6651 NewCall->takeName(CI);
6652 CI->replaceAllUsesWith(NewCall);
6653 CI->eraseFromParent();
6654}
6655
6656void llvm::UpgradeCallsToIntrinsic(Function *F) {
6657 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6658
6659 // Check if this function should be upgraded and get the replacement function
6660 // if there is one.
6661 Function *NewFn;
6662 if (UpgradeIntrinsicFunction(F, NewFn)) {
6663 // Replace all users of the old function with the new function or new
6664 // instructions. This is not a range loop because the call is deleted.
6665 for (User *U : make_early_inc_range(F->users()))
6666 if (CallBase *CB = dyn_cast<CallBase>(U))
6667 UpgradeIntrinsicCall(CB, NewFn);
6668
6669 // Remove old function, no longer used, from the module.
6670 if (F != NewFn)
6671 F->eraseFromParent();
6672 }
6673}
6674
6675MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6676 const unsigned NumOperands = MD.getNumOperands();
6677 if (NumOperands == 0)
6678 return &MD; // Invalid, punt to a verifier error.
6679
6680 // Check if the tag uses struct-path aware TBAA format.
6681 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6682 return &MD;
6683
6684 auto &Context = MD.getContext();
6685 if (NumOperands == 3) {
6686 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6687 MDNode *ScalarType = MDNode::get(Context, Elts);
6688 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6689 Metadata *Elts2[] = {ScalarType, ScalarType,
6690 ConstantAsMetadata::get(
6691 Constant::getNullValue(Type::getInt64Ty(Context))),
6692 MD.getOperand(2)};
6693 return MDNode::get(Context, Elts2);
6694 }
6695 // Create a MDNode <MD, MD, offset 0>
6696 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6697 Type::getInt64Ty(Context)))};
6698 return MDNode::get(Context, Elts);
6699}
6700
6701MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6702 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6703 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6704 unsigned NumOperands = MD.getNumOperands();
6705 if (NumOperands == 0 || NumOperands % 3 != 0)
6706 return &MD; // Malformed; leave it for the verifier to reject.
6707
6708 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6709 bool Changed = false;
6710 for (unsigned I = 2; I < NumOperands; I += 3) {
6711 auto *Tag = dyn_cast_or_null<MDNode>(Elts[I]);
6712 if (!Tag)
6713 continue;
6714 MDNode *Upgraded = UpgradeTBAANode(*Tag);
6715 if (Upgraded == Tag)
6716 continue;
6717 Elts[I] = Upgraded;
6718 Changed = true;
6719 }
6720 return Changed ? MDNode::get(MD.getContext(), Elts) : &MD;
6721}
6722
6723Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6724 Instruction *&Temp) {
6725 if (Opc != Instruction::BitCast)
6726 return nullptr;
6727
6728 Temp = nullptr;
6729 Type *SrcTy = V->getType();
6730 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6731 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6732 LLVMContext &Context = V->getContext();
6733
6734 // We have no information about target data layout, so we assume that
6735 // the maximum pointer size is 64bit.
6736 Type *MidTy = Type::getInt64Ty(Context);
6737 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6738
6739 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6740 }
6741
6742 return nullptr;
6743}
6744
6745Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6746 if (Opc != Instruction::BitCast)
6747 return nullptr;
6748
6749 Type *SrcTy = C->getType();
6750 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6751 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6752 LLVMContext &Context = C->getContext();
6753
6754 // We have no information about target data layout, so we assume that
6755 // the maximum pointer size is 64bit.
6756 Type *MidTy = Type::getInt64Ty(Context);
6757
6758 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6759 DestTy);
6760 }
6761
6762 return nullptr;
6763}
6764
6765static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6766 if (Flag.getNumOperands() < 3)
6767 return std::nullopt;
6768 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6769 return Name->getString();
6770 return std::nullopt;
6771}
6772
6773/// Check the debug info version number, if it is out-dated, drop the debug
6774/// info. Return true if module is modified.
6775bool llvm::UpgradeDebugInfo(Module &M) {
6776 if (!getIROptions(M.getContext()).auto_upgrade_debug_info)
6777 return false;
6778
6779 llvm::TimeTraceScope timeScope("Upgrade debug info");
6780 // We need to get metadata before the module is verified (i.e., getModuleFlag
6781 // makes assumptions that we haven't verified yet). Carefully extract the flag
6782 // from the metadata.
6783 unsigned Version = 0;
6784 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6785 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6786 if (auto Name = getModuleFlagNameSafely(*Flag))
6787 return *Name == "Debug Info Version";
6788 return false;
6789 });
6790 if (OpIt != ModFlags->op_end()) {
6791 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6792 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6793 Version = CI->getZExtValue();
6794 }
6795 }
6796
6797 if (Version == DEBUG_METADATA_VERSION) {
6798 bool BrokenDebugInfo = false;
6799 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6800 report_fatal_error("Broken module found, compilation aborted!");
6801 if (!BrokenDebugInfo)
6802 // Everything is ok.
6803 return false;
6804 else {
6805 // Diagnose malformed debug info.
6806 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6807 M.getContext().diagnose(Diag);
6808 }
6809 }
6810 bool Modified = StripDebugInfo(M);
6811 if (Modified && Version != DEBUG_METADATA_VERSION) {
6812 // Diagnose a version mismatch.
6813 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6814 M.getContext().diagnose(DiagVersion);
6815 }
6816 return Modified;
6817}
6818
6819static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6820 GlobalValue *GV, const Metadata *V) {
6821 Function *F = cast<Function>(GV);
6822
6823 constexpr StringLiteral DefaultValue = "1";
6824 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6825 unsigned Length = 0;
6826
6827 if (F->hasFnAttribute(Attr)) {
6828 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6829 // parse these elements placing them into Vect3
6830 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6831 for (; Length < 3 && !S.empty(); Length++) {
6832 auto [Part, Rest] = S.split(',');
6833 Vect3[Length] = Part.trim();
6834 S = Rest;
6835 }
6836 }
6837
6838 const unsigned Dim = DimC - 'x';
6839 assert(Dim < 3 && "Unexpected dim char");
6840
6841 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6842
6843 // local variable required for StringRef in Vect3 to point to.
6844 const std::string VStr = llvm::utostr(VInt);
6845 Vect3[Dim] = VStr;
6846 Length = std::max(Length, Dim + 1);
6847
6848 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6849 F->addFnAttr(Attr, NewAttr);
6850}
6851
6852static inline bool isXYZ(StringRef S) {
6853 return S == "x" || S == "y" || S == "z";
6854}
6855
6856bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6857 const Metadata *V) {
6858 if (K == "kernel") {
6859 if (!mdconst::extract<ConstantInt>(V)->isZero())
6860 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6861 return true;
6862 }
6863 if (K == "align") {
6864 // V is a bitfeild specifying two 16-bit values. The alignment value is
6865 // specfied in low 16-bits, The index is specified in the high bits. For the
6866 // index, 0 indicates the return value while higher values correspond to
6867 // each parameter (idx = param + 1).
6868 const uint64_t AlignIdxValuePair =
6869 mdconst::extract<ConstantInt>(V)->getZExtValue();
6870 const unsigned Idx = (AlignIdxValuePair >> 16);
6871 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6872 cast<Function>(GV)->addAttributeAtIndex(
6873 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6874 return true;
6875 }
6876 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6877 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6878 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6879 return true;
6880 }
6881 if (K == "minctasm") {
6882 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6883 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6884 return true;
6885 }
6886 if (K == "maxnreg") {
6887 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6888 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6889 return true;
6890 }
6891 if (K.consume_front("maxntid") && isXYZ(K)) {
6892 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6893 return true;
6894 }
6895 if (K.consume_front("reqntid") && isXYZ(K)) {
6896 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6897 return true;
6898 }
6899 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6900 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6901 return true;
6902 }
6903 if (K == "grid_constant") {
6904 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6905 for (const auto &Op : cast<MDNode>(V)->operands()) {
6906 // For some reason, the index is 1-based in the metadata. Good thing we're
6907 // able to auto-upgrade it!
6908 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6909 cast<Function>(GV)->addParamAttr(Index, Attr);
6910 }
6911 return true;
6912 }
6913
6914 return false;
6915}
6916
6917void llvm::UpgradeNVVMAnnotations(Module &M) {
6918 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6919 if (!NamedMD)
6920 return;
6921
6922 SmallVector<MDNode *, 8> NewNodes;
6923 SmallPtrSet<const MDNode *, 8> SeenNodes;
6924 for (MDNode *MD : NamedMD->operands()) {
6925 if (!SeenNodes.insert(MD).second)
6926 continue;
6927
6928 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6929 if (!GV)
6930 continue;
6931
6932 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6933
6934 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6935 // Each nvvm.annotations metadata entry will be of the following form:
6936 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6937 // start index = 1, to skip the global variable key
6938 // increment = 2, to skip the value for each property-value pairs
6939 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6940 MDString *K = cast<MDString>(MD->getOperand(j));
6941 const MDOperand &V = MD->getOperand(j + 1);
6942 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6943 if (!Upgraded)
6944 NewOperands.append({K, V});
6945 }
6946
6947 if (NewOperands.size() > 1)
6948 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6949 }
6950
6951 NamedMD->clearOperands();
6952 for (MDNode *N : NewNodes)
6953 NamedMD->addOperand(N);
6954}
6955
6956/// This checks for objc retain release marker which should be upgraded. It
6957/// returns true if module is modified.
6958static bool upgradeRetainReleaseMarker(Module &M) {
6959 bool Changed = false;
6960 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6961 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6962 if (ModRetainReleaseMarker) {
6963 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6964 if (Op) {
6965 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6966 if (ID) {
6967 SmallVector<StringRef, 4> ValueComp;
6968 ID->getString().split(ValueComp, "#");
6969 if (ValueComp.size() == 2) {
6970 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6971 ID = MDString::get(M.getContext(), NewValue);
6972 }
6973 M.addModuleFlag(Module::Error, MarkerKey, ID);
6974 M.eraseNamedMetadata(ModRetainReleaseMarker);
6975 Changed = true;
6976 }
6977 }
6978 }
6979 return Changed;
6980}
6981
6982void llvm::UpgradeARCRuntime(Module &M) {
6983 // This lambda converts normal function calls to ARC runtime functions to
6984 // intrinsic calls.
6985 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6986 llvm::Intrinsic::ID IntrinsicFunc) {
6987 Function *Fn = M.getFunction(OldFunc);
6988
6989 if (!Fn)
6990 return;
6991
6992 Function *NewFn =
6993 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6994
6995 for (User *U : make_early_inc_range(Fn->users())) {
6996 CallInst *CI = dyn_cast<CallInst>(U);
6997 if (!CI || CI->getCalledFunction() != Fn)
6998 continue;
6999
7000 IRBuilder<> Builder(CI->getIterator());
7001 FunctionType *NewFuncTy = NewFn->getFunctionType();
7002 SmallVector<Value *, 2> Args;
7003
7004 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7005 // value to the return type of the old function.
7006 if (NewFuncTy->getReturnType() != CI->getType() &&
7007 !CastInst::castIsValid(Instruction::BitCast, CI,
7008 NewFuncTy->getReturnType()))
7009 continue;
7010
7011 bool InvalidCast = false;
7012
7013 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7014 Value *Arg = CI->getArgOperand(I);
7015
7016 // Bitcast argument to the parameter type of the new function if it's
7017 // not a variadic argument.
7018 if (I < NewFuncTy->getNumParams()) {
7019 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7020 // to the parameter type of the new function.
7021 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
7022 NewFuncTy->getParamType(I))) {
7023 InvalidCast = true;
7024 break;
7025 }
7026 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
7027 }
7028 Args.push_back(Arg);
7029 }
7030
7031 if (InvalidCast)
7032 continue;
7033
7034 // Create a call instruction that calls the new function.
7035 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7036 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
7037 NewCall->takeName(CI);
7038
7039 // Bitcast the return value back to the type of the old call.
7040 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
7041
7042 if (!CI->use_empty())
7043 CI->replaceAllUsesWith(NewRetVal);
7044 CI->eraseFromParent();
7045 }
7046
7047 if (Fn->use_empty())
7048 Fn->eraseFromParent();
7049 };
7050
7051 // Unconditionally convert a call to "clang.arc.use" to a call to
7052 // "llvm.objc.clang.arc.use".
7053 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7054
7055 // Upgrade the retain release marker. If there is no need to upgrade
7056 // the marker, that means either the module is already new enough to contain
7057 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7058 if (!upgradeRetainReleaseMarker(M))
7059 return;
7060
7061 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7062 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7063 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7064 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7065 {"objc_autoreleaseReturnValue",
7066 llvm::Intrinsic::objc_autoreleaseReturnValue},
7067 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7068 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7069 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7070 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7071 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7072 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7073 {"objc_release", llvm::Intrinsic::objc_release},
7074 {"objc_retain", llvm::Intrinsic::objc_retain},
7075 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7076 {"objc_retainAutoreleaseReturnValue",
7077 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7078 {"objc_retainAutoreleasedReturnValue",
7079 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7080 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7081 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7082 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7083 {"objc_unsafeClaimAutoreleasedReturnValue",
7084 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7085 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7086 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7087 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7088 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7089 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7090 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7091 {"objc_arc_annotation_topdown_bbstart",
7092 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7093 {"objc_arc_annotation_topdown_bbend",
7094 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7095 {"objc_arc_annotation_bottomup_bbstart",
7096 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7097 {"objc_arc_annotation_bottomup_bbend",
7098 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7099
7100 for (auto &I : RuntimeFuncs)
7101 UpgradeToIntrinsic(I.first, I.second);
7102}
7103
7104// Upgrade the way signing of pointers to init/fini functions is described.
7105//
7106// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7107// constants, if signing was requested. After the upgrade, these arrays contain
7108// plain function pointers and the desired signing schema is described via a
7109// pair of module flags.
7110//
7111// Note that the upgrade is only performed if all elements of *both* arrays
7112// agree on a common signing schema.
7113static bool upgradePtrauthInitFiniArrays(Module &M) {
7114 // As we cannot always decide whether the particular module should have
7115 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7116 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7117 // returns as soon as it spots any non-signed init/fini pointer: either we
7118 // should request non-signed pointers (safe to omit both flags) or there is
7119 // no common schema (and thus we do not modify anything).
7120 //
7121 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7122 // or whether we should request address diversity in addition to the basic
7123 // constant diversity. There is no value representing "decided not to sign"
7124 // for the reasons explained above.
7125 std::optional<bool> UseAddressDisc;
7126
7127 // Do not attempt upgrading if the new module flags already exist.
7128 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7129 for (const MDNode *Flag : ModFlags->operands()) {
7130 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7131 if (Name && (*Name == "ptrauth-init-fini" ||
7132 *Name == "ptrauth-init-fini-address-discrimination"))
7133 return false;
7134 }
7135 }
7136
7137 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7138 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7139 constexpr unsigned ExpectedAddressMarker = 1;
7140
7141 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7142 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7143 return nullptr; // Nothing to upgrade or unknown pattern found.
7144
7145 bool HasAddressDisc;
7146 if (!CPA->hasAddressDiscriminator())
7147 HasAddressDisc = false;
7148 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7149 HasAddressDisc = true;
7150 else
7151 return nullptr; // Unknown pattern.
7152
7153 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7154 return nullptr; // Disagreement with the decided mode.
7155
7156 UseAddressDisc = HasAddressDisc;
7157 return CPA->getPointer();
7158 };
7159
7160 // Do not apply any changes until we know the upgrade is non-ambiguous.
7161 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7162 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7163
7164 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7165 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7166 if (!GV || !GV->hasInitializer())
7167 continue; // Skip, but it is okay to upgrade the other variable.
7168
7169 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7170 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7171 return false;
7172
7173 std::vector<Constant *> NewStructors;
7174 NewStructors.reserve(OldStructorsArray->getNumOperands());
7175
7176 for (Use &U : OldStructorsArray->operands()) {
7177 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7178 if (!Structor || Structor->getNumOperands() != 3)
7179 return false;
7180
7181 Constant *Prio = Structor->getOperand(0);
7182 Constant *Func = Structor->getOperand(1);
7183 Constant *Arg = Structor->getOperand(2);
7184
7185 Func = UpgradeSinglePointer(Func);
7186 if (!Func)
7187 return false;
7188
7189 NewStructors.push_back(
7190 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7191 }
7192
7193 Constant *NewInit =
7194 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7195 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7196 }
7197
7198 if (GlobalArraysToUpgrade.empty())
7199 return false;
7200 assert(UseAddressDisc.has_value());
7201
7202 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7203 GV->setInitializer(NewInit);
7204
7205 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7206 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7207 *UseAddressDisc);
7208
7209 return true;
7210}
7211
7212bool llvm::UpgradeModuleFlags(Module &M) {
7213 bool Changed = false;
7214 Changed |= upgradePtrauthInitFiniArrays(M);
7215
7216 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7217 if (!ModFlags)
7218 return Changed;
7219
7220 bool HasObjCFlag = false, HasClassProperties = false;
7221 bool HasSwiftVersionFlag = false;
7222 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7223 uint32_t SwiftABIVersion;
7224 auto Int8Ty = Type::getInt8Ty(M.getContext());
7225 auto Int32Ty = Type::getInt32Ty(M.getContext());
7226
7227 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7228 MDNode *Op = ModFlags->getOperand(I);
7229 if (Op->getNumOperands() != 3)
7230 continue;
7231 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7232 if (!ID)
7233 continue;
7234 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7235 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7236 Type::getInt32Ty(M.getContext()), B)),
7237 MDString::get(M.getContext(), ID->getString()),
7238 Op->getOperand(2)};
7239 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7240 Changed = true;
7241 };
7242
7243 if (ID->getString() == "Objective-C Image Info Version")
7244 HasObjCFlag = true;
7245 if (ID->getString() == "Objective-C Class Properties")
7246 HasClassProperties = true;
7247 // Upgrade PIC from Error/Max to Min.
7248 if (ID->getString() == "PIC Level") {
7249 if (auto *Behavior =
7250 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7251 uint64_t V = Behavior->getLimitedValue();
7252 if (V == Module::Error || V == Module::Max)
7253 SetBehavior(Module::Min);
7254 }
7255 }
7256 // Upgrade "PIE Level" from Error to Max.
7257 if (ID->getString() == "PIE Level")
7258 if (auto *Behavior =
7259 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7260 if (Behavior->getLimitedValue() == Module::Error)
7261 SetBehavior(Module::Max);
7262
7263 // Upgrade branch protection and return address signing module flags. The
7264 // module flag behavior for these fields were Error and now they are Min.
7265 // The one exception is "sign-return-address-harden".
7266 if (ID->getString() == "branch-target-enforcement" ||
7267 (ID->getString().starts_with("sign-return-address") &&
7268 ID->getString() != "sign-return-address-harden")) {
7269 if (auto *Behavior =
7270 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7271 if (Behavior->getLimitedValue() == Module::Error) {
7272 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7273 Metadata *Ops[3] = {
7274 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7275 Op->getOperand(1), Op->getOperand(2)};
7276 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7277 Changed = true;
7278 }
7279 }
7280 }
7281
7282 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7283 // section name so that llvm-lto will not complain about mismatching
7284 // module flags that is functionally the same.
7285 if (ID->getString() == "Objective-C Image Info Section") {
7286 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7287 SmallVector<StringRef, 4> ValueComp;
7288 Value->getString().split(ValueComp, " ");
7289 if (ValueComp.size() != 1) {
7290 std::string NewValue;
7291 for (auto &S : ValueComp)
7292 NewValue += S.str();
7293 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7294 MDString::get(M.getContext(), NewValue)};
7295 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7296 Changed = true;
7297 }
7298 }
7299 }
7300
7301 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7302 // If the higher bits are set, it adds new module flag for swift info.
7303 if (ID->getString() == "Objective-C Garbage Collection") {
7304 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7305 if (Md) {
7306 assert(Md->getValue() && "Expected non-empty metadata");
7307 auto Type = Md->getValue()->getType();
7308 if (Type == Int8Ty)
7309 continue;
7310 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7311 if ((Val & 0xff) != Val) {
7312 HasSwiftVersionFlag = true;
7313 SwiftABIVersion = (Val & 0xff00) >> 8;
7314 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7315 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7316 }
7317 Metadata *Ops[3] = {
7318 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7319 Op->getOperand(1),
7320 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7321 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7322 Changed = true;
7323 }
7324 }
7325
7326 if (ID->getString() == "amdgpu_code_object_version") {
7327 Metadata *Ops[3] = {
7328 Op->getOperand(0),
7329 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7330 Op->getOperand(2)};
7331 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7332 Changed = true;
7333 }
7334
7335 // clang/PowerPC used to use "float-abi" to describe the long double format;
7336 // it has been renamed to "long-double-type", with its values changed to the
7337 // corresponding IR floating-point type names.
7338 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7339 StringRef Format;
7340 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7341 Format = S->getString();
7342
7343 // The "float-abi" key is now reserved for the target-independent
7344 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7345 // (including unrecognized ones, which were never valid) to the default.
7346 if (!FloatABI::parseABIType(Format)) {
7347 LongDoubleFormat NewFormat =
7348 StringSwitch<LongDoubleFormat>(Format)
7349 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7350 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7351 .Default(LongDoubleFormat::PPCDoubleDouble);
7352 Metadata *Ops[3] = {
7353 Op->getOperand(0),
7354 MDString::get(M.getContext(), "long-double-type"),
7355 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7356 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7357 Changed = true;
7358 }
7359 }
7360 }
7361
7362 // "Objective-C Class Properties" is recently added for Objective-C. We
7363 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7364 // flag of value 0, so we can correclty downgrade this flag when trying to
7365 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7366 // this module flag.
7367 if (HasObjCFlag && !HasClassProperties) {
7368 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7369 (uint32_t)0);
7370 Changed = true;
7371 }
7372
7373 if (HasSwiftVersionFlag) {
7374 M.addModuleFlag(Module::Error, "Swift ABI Version",
7375 SwiftABIVersion);
7376 M.addModuleFlag(Module::Error, "Swift Major Version",
7377 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7378 M.addModuleFlag(Module::Error, "Swift Minor Version",
7379 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7380 Changed = true;
7381 }
7382
7383 return Changed;
7384}
7385
7386bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7387 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7388 // If this metadata has operands, we expect all of them to be either from
7389 // before or from after the format change handled here, so we can bail out
7390 // fast if the first (if any) operands is of the new format.
7391 auto MatchesVersion = [](const MDNode *Op) {
7392 return Op->getNumOperands() >= 3 &&
7393 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7394 cast<ConstantAsMetadata>(Op->getOperand(2))
7395 ->getType()
7396 ->isIntegerTy(64);
7397 };
7398
7399 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7400 MatchesVersion(CFIConsts->getOperand(0)))
7401 return false;
7402
7403 bool Changed = false;
7404 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7405 MDNode *Op = CFIConsts->getOperand(I);
7406 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7407 assert(Op->getNumOperands() >= 2 &&
7408 "Expected at least 2 operands - name and linkage type");
7409 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7410 StringRef Name = NameMD->getString();
7411 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7412 GlobalValue::dropLLVMManglingEscape(Name));
7413
7414 SmallVector<Metadata *, 4> Elts;
7415 Elts.push_back(Op->getOperand(0));
7416 Elts.push_back(Op->getOperand(1));
7417 Elts.push_back(ConstantAsMetadata::get(
7418 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7419
7420 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7421 Elts.push_back(Op->getOperand(J));
7422
7423 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7424 Changed = true;
7425 }
7426
7427 return Changed;
7428}
7429
7430void llvm::UpgradeSectionAttributes(Module &M) {
7431 auto TrimSpaces = [](StringRef Section) -> std::string {
7432 SmallVector<StringRef, 5> Components;
7433 Section.split(Components, ',');
7434
7435 SmallString<32> Buffer;
7436 raw_svector_ostream OS(Buffer);
7437
7438 for (auto Component : Components)
7439 OS << ',' << Component.trim();
7440
7441 return std::string(OS.str().substr(1));
7442 };
7443
7444 for (auto &GV : M.globals()) {
7445 if (!GV.hasSection())
7446 continue;
7447
7448 StringRef Section = GV.getSection();
7449
7450 if (!Section.starts_with("__DATA, __objc_catlist"))
7451 continue;
7452
7453 // __DATA, __objc_catlist, regular, no_dead_strip
7454 // __DATA,__objc_catlist,regular,no_dead_strip
7455 GV.setSection(TrimSpaces(Section));
7456 }
7457}
7458
7459namespace {
7460// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7461// callsites within a function that did not also have the strictfp attribute.
7462// Since 10.0, if strict FP semantics are needed within a function, the
7463// function must have the strictfp attribute and all calls within the function
7464// must also have the strictfp attribute. This latter restriction is
7465// necessary to prevent unwanted libcall simplification when a function is
7466// being cloned (such as for inlining).
7467//
7468// The "dangling" strictfp attribute usage was only used to prevent constant
7469// folding and other libcall simplification. The nobuiltin attribute on the
7470// callsite has the same effect.
7471struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7472 StrictFPUpgradeVisitor() = default;
7473
7474 void visitCallBase(CallBase &Call) {
7475 if (!Call.isStrictFP())
7476 return;
7477 if (isa<ConstrainedFPIntrinsic>(&Call))
7478 return;
7479 // If we get here, the caller doesn't have the strictfp attribute
7480 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7481 Call.removeFnAttr(Attribute::StrictFP);
7482 Call.addFnAttr(Attribute::NoBuiltin);
7483 }
7484};
7485
7486/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7487struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7488 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7489 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7490
7491 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7492 if (!RMW.isFloatingPointOperation())
7493 return;
7494
7495 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7496 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7497 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7498 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7499 }
7500};
7501} // namespace
7502
7503void llvm::UpgradeFunctionAttributes(Function &F) {
7504 // If a function definition doesn't have the strictfp attribute,
7505 // convert any callsite strictfp attributes to nobuiltin.
7506 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7507 StrictFPUpgradeVisitor SFPV;
7508 SFPV.visit(F);
7509 }
7510
7511 // Remove all incompatibile attributes from function.
7512 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7513 F.getReturnType(), F.getAttributes().getRetAttrs()));
7514 for (auto &Arg : F.args())
7515 Arg.removeAttrs(
7516 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7517
7518 bool AddingAttrs = false, RemovingAttrs = false;
7519 AttrBuilder AttrsToAdd(F.getContext());
7520 AttributeMask AttrsToRemove;
7521
7522 // Older versions of LLVM treated an "implicit-section-name" attribute
7523 // similarly to directly setting the section on a Function.
7524 if (Attribute A = F.getFnAttribute("implicit-section-name");
7525 A.isValid() && A.isStringAttribute()) {
7526 F.setSection(A.getValueAsString());
7527 AttrsToRemove.addAttribute("implicit-section-name");
7528 RemovingAttrs = true;
7529 }
7530
7531 if (Attribute A = F.getFnAttribute("nooutline");
7532 A.isValid() && A.isStringAttribute()) {
7533 AttrsToRemove.addAttribute("nooutline");
7534 AttrsToAdd.addAttribute(Attribute::NoOutline);
7535 AddingAttrs = RemovingAttrs = true;
7536 }
7537
7538 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7539 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7540 AttrsToRemove.addAttribute("uniform-work-group-size");
7541 RemovingAttrs = true;
7542 if (A.getValueAsString() == "true") {
7543 AttrsToAdd.addAttribute("uniform-work-group-size");
7544 AddingAttrs = true;
7545 }
7546 }
7547
7548 if (!F.empty()) {
7549 // For some reason this is called twice, and the first time is before any
7550 // instructions are loaded into the body.
7551
7552 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7553 A.isValid()) {
7554
7555 if (A.getValueAsBool()) {
7556 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7557 Visitor.visit(F);
7558 }
7559
7560 // We will leave behind dead attribute uses on external declarations, but
7561 // clang never added these to declarations anyway.
7562 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7563 RemovingAttrs = true;
7564 }
7565 }
7566
7567 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7568 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7569
7570 bool HandleDenormalMode = false;
7571
7572 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7573 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7574 if (ParsedMode.isValid()) {
7575 DenormalFPMath = ParsedMode;
7576 AttrsToRemove.addAttribute("denormal-fp-math");
7577 AddingAttrs = RemovingAttrs = true;
7578 HandleDenormalMode = true;
7579 }
7580 }
7581
7582 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7583 Attr.isValid()) {
7584 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7585 if (ParsedMode.isValid()) {
7586 DenormalFPMathF32 = ParsedMode;
7587 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7588 AddingAttrs = RemovingAttrs = true;
7589 HandleDenormalMode = true;
7590 }
7591 }
7592
7593 if (HandleDenormalMode)
7594 AttrsToAdd.addDenormalFPEnvAttr(
7595 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7596
7597 if (RemovingAttrs)
7598 F.removeFnAttrs(AttrsToRemove);
7599
7600 if (AddingAttrs)
7601 F.addFnAttrs(AttrsToAdd);
7602}
7603
7604// Check if the function attribute is not present and set it.
7605static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7606 StringRef Value) {
7607 if (!F.hasFnAttribute(FnAttrName)) {
7608 F.addFnAttr(FnAttrName, Value);
7609 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7610 << "\", function: " << F.getName() << "\n");
7611 }
7612}
7613
7614// Check if the function attribute is not present and set it if needed.
7615// If the attribute is "false" then removes it.
7616// If the attribute is "true" resets it to a valueless attribute.
7617static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7618 if (!F.hasFnAttribute(FnAttrName)) {
7619 if (Set) {
7620 F.addFnAttr(FnAttrName);
7621 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7622 << ", function: " << F.getName() << "\n");
7623 }
7624 } else {
7625 auto A = F.getFnAttribute(FnAttrName);
7626 if ("false" == A.getValueAsString()) {
7627 F.removeFnAttr(FnAttrName);
7628 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7629 << "=\"false\", function: " << F.getName() << "\n");
7630 } else if ("true" == A.getValueAsString()) {
7631 F.removeFnAttr(FnAttrName);
7632 F.addFnAttr(FnAttrName);
7633 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7634 << "=\"true\", function: " << F.getName() << "\n");
7635 }
7636 }
7637}
7638
7639static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7640 StringRef Key, uint32_t Val) {
7641 M.setModuleFlag(Behavior, Key, Val);
7642 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7643 << Key << ", " << Val << "}\n");
7644}
7645
7646void llvm::copyModuleAttrToFunctions(Module &M) {
7647 Triple T(M.getTargetTriple());
7648 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7649 return;
7650
7651 uint64_t BTEValue = 0;
7652 uint64_t BPPLRValue = 0;
7653 uint64_t GCSValue = 0;
7654 uint64_t SRAValue = 0;
7655 uint64_t SRAALLValue = 0;
7656 uint64_t SRABKeyValue = 0;
7657
7658 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7659 if (ModFlags) {
7660 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7661 MDNode *Op = ModFlags->getOperand(I);
7662 if (Op->getNumOperands() != 3)
7663 continue;
7664
7665 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7666 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7667 if (!ID || !CI)
7668 continue;
7669
7670 StringRef IDStr = ID->getString();
7671 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7672 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7673 : IDStr == "guarded-control-stack" ? &GCSValue
7674 : IDStr == "sign-return-address" ? &SRAValue
7675 : IDStr == "sign-return-address-all" ? &SRAALLValue
7676 : IDStr == "sign-return-address-with-bkey"
7677 ? &SRABKeyValue
7678 : nullptr;
7679 if (!ValPtr)
7680 continue;
7681
7682 *ValPtr = CI->getZExtValue();
7683 if (*ValPtr == 2)
7684 return;
7685
7686 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7687 << ")\n");
7688 }
7689 }
7690
7691 bool BTE = BTEValue == 1;
7692 bool BPPLR = BPPLRValue == 1;
7693 bool GCS = GCSValue == 1;
7694 bool SRA = SRAValue == 1;
7695
7696 StringRef SignTypeValue = "non-leaf";
7697 if (SRA && SRAALLValue == 1)
7698 SignTypeValue = "all";
7699
7700 StringRef SignKeyValue = "a_key";
7701 if (SRA && SRABKeyValue == 1)
7702 SignKeyValue = "b_key";
7703
7704 for (Function &F : M.getFunctionList()) {
7705 if (F.isDeclaration())
7706 continue;
7707
7708 if (SRA) {
7709 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7710 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7711 } else {
7712 if (auto A = F.getFnAttribute("sign-return-address");
7713 A.isValid() && "none" == A.getValueAsString()) {
7714 F.removeFnAttr("sign-return-address");
7715 F.removeFnAttr("sign-return-address-key");
7716 }
7717 }
7718 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7719 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7720 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7721 }
7722
7723 if (BTE)
7724 ConvertModuleFlag(M, llvm::Module::Min, "branch-target-enforcement", 2);
7725 if (BPPLR)
7726 ConvertModuleFlag(M, llvm::Module::Min, "branch-protection-pauth-lr", 2);
7727 if (GCS)
7728 ConvertModuleFlag(M, llvm::Module::Min, "guarded-control-stack", 2);
7729 if (SRA) {
7730 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address", 2);
7731 if (SRAALLValue == 1)
7732 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-all", 2);
7733 if (SRABKeyValue == 1) {
7734 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-with-bkey",
7735 2);
7736 }
7737 }
7738}
7739
7740/// Return the replacement tags if \p T still uses a removed two-operand form.
7741static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7742 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7743 return nullptr;
7744 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7745 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7746}
7747
7748/// Build the single-operand node that replaces a boolean operand: nonzero
7749/// selects the enable tag, zero the disable tag.
7750static Metadata *makeBooleanLoopNode(LLVMContext &C,
7751 const BooleanLoopTags &Tags,
7752 const MDOperand &Op) {
7753 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7754 return MDTuple::get(C,
7755 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7756}
7757
7758static bool isOldLoopArgument(Metadata *MD) {
7759 auto *T = dyn_cast_or_null<MDTuple>(MD);
7760 if (!T)
7761 return false;
7762 if (T->getNumOperands() < 1)
7763 return false;
7764 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7765 if (!S)
7766 return false;
7767 if (S->getString().starts_with("llvm.vectorizer."))
7768 return true;
7769 return getOldBooleanLoopTags(T) != nullptr;
7770}
7771
7772static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7773 StringRef OldPrefix = "llvm.vectorizer.";
7774 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7775
7776 if (OldTag == "llvm.vectorizer.unroll")
7777 return MDString::get(C, "llvm.loop.interleave.count");
7778
7779 return MDString::get(
7780 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7781 .str());
7782}
7783
7784static Metadata *upgradeLoopArgument(Metadata *MD) {
7785 auto *T = dyn_cast_or_null<MDTuple>(MD);
7786 if (!T)
7787 return MD;
7788 if (T->getNumOperands() < 1)
7789 return MD;
7790 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7791 if (!OldTag)
7792 return MD;
7793
7794 LLVMContext &C = T->getContext();
7795
7796 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7797 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7798 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7799
7800 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7801 return MD;
7802
7803 // This has an old tag. Upgrade it.
7804 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7805
7806 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7807 // vectorize.enable/disable pair, not a two-operand enable node.
7808 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7809 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7810 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7811
7812 SmallVector<Metadata *, 8> Ops;
7813 Ops.reserve(T->getNumOperands());
7814 Ops.push_back(NewTag);
7815 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7816 Ops.push_back(T->getOperand(I));
7817
7818 return MDTuple::get(C, Ops);
7819}
7820
7821MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7822 auto *T = dyn_cast<MDTuple>(&N);
7823 if (!T)
7824 return &N;
7825
7826 if (none_of(T->operands(), isOldLoopArgument))
7827 return &N;
7828
7829 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7830 // any MDNode carrying those tags with more than one operand, so a leftover
7831 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7832 // In-place mutation is safe on distinct MDNodes.
7833 if (T->isDistinct()) {
7834 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7835 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7836 if (OpT && getOldBooleanLoopTags(OpT))
7837 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7838 }
7839 if (none_of(T->operands(), isOldLoopArgument))
7840 return &N;
7841 }
7842
7843 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7844 // attachment; the original distinct loop-ID is kept as the first operand.
7845 SmallVector<Metadata *, 8> Ops;
7846 Ops.reserve(T->getNumOperands());
7847 for (Metadata *MD : T->operands())
7848 Ops.push_back(upgradeLoopArgument(MD));
7849
7850 return MDTuple::get(T->getContext(), Ops);
7851}
7852
7853std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7854 Triple T(TT);
7855 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7856 // the address space of globals to 1. This does not apply to SPIRV Logical.
7857 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7858 !DL.contains("-G") && !DL.starts_with("G")) {
7859 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7860 }
7861
7862 if (T.isLoongArch64() || T.isRISCV64()) {
7863 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7864 auto I = DL.find("-n64-");
7865 if (I != StringRef::npos)
7866 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7867 return DL.str();
7868 }
7869
7870 // AMDGPU data layout upgrades.
7871 std::string Res = DL.str();
7872 if (T.isAMDGPU()) {
7873 // Define address spaces for constants.
7874 if (!DL.contains("-G") && !DL.starts_with("G"))
7875 Res.append(Res.empty() ? "G1" : "-G1");
7876
7877 // AMDGCN data layout upgrades.
7878 if (T.isAMDGCN()) {
7879
7880 // Add missing non-integral declarations.
7881 // This goes before adding new address spaces to prevent incoherent string
7882 // values.
7883 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7884 Res.append("-ni:7:8:9");
7885 // Update ni:7 to ni:7:8:9.
7886 if (DL.ends_with("ni:7"))
7887 Res.append(":8:9");
7888 if (DL.ends_with("ni:7:8"))
7889 Res.append(":9");
7890
7891 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7892 // resources) An empty data layout has already been upgraded to G1 by now.
7893 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7894 Res.append("-p7:160:256:256:32");
7895 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7896 Res.append("-p8:128:128:128:48");
7897 constexpr StringRef OldP8("-p8:128:128-");
7898 if (DL.contains(OldP8))
7899 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7900 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7901 Res.append("-p9:192:256:256:32");
7902
7903 // Add sizing for address space 10 through 15.
7904 // AS 10-14 are reserved and defaulted to 32:32
7905 // AS 15 is in use and is 32:32.
7906 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7907 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7908 Res.append(("-" + AS + ":32:32").str());
7909 }
7910 }
7911
7912 // Upgrade the ELF mangling mode.
7913 if (!DL.contains("m:e"))
7914 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7915
7916 return Res;
7917 }
7918
7919 if (T.isSystemZ() && !DL.empty()) {
7920 // Make sure the stack alignment is present.
7921 if (!DL.contains("-S64"))
7922 return "E-S64" + DL.drop_front(1).str();
7923 return DL.str();
7924 }
7925
7926 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7927 // If the datalayout matches the expected format, add pointer size address
7928 // spaces to the datalayout.
7929 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7930 if (!DL.contains(AddrSpaces)) {
7931 SmallVector<StringRef, 4> Groups;
7932 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7933 if (R.match(Res, &Groups))
7934 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7935 }
7936 };
7937
7938 // AArch64 data layout upgrades.
7939 if (T.isAArch64()) {
7940 // Add "-Fn32"
7941 if (!DL.empty() && !DL.contains("-Fn32"))
7942 Res.append("-Fn32");
7943 AddPtr32Ptr64AddrSpaces();
7944 return Res;
7945 }
7946
7947 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7948 T.isWasm()) {
7949 // Mips64 with o32 ABI did not add "-i128:128".
7950 // Add "-i128:128"
7951 std::string I64 = "-i64:64";
7952 std::string I128 = "-i128:128";
7953 if (!StringRef(Res).contains(I128)) {
7954 size_t Pos = Res.find(I64);
7955 if (Pos != size_t(-1))
7956 Res.insert(Pos + I64.size(), I128);
7957 }
7958 }
7959
7960 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7961 size_t Pos = Res.find("-S128");
7962 if (Pos == StringRef::npos)
7963 Pos = Res.size();
7964 Res.insert(Pos, "-f64:32:64");
7965 }
7966
7967 // ARM data layout upgrades.
7968 // Add -Fi8 if a -F has not already been specified.
7969 if (T.isARM() && !DL.empty() && !DL.contains("Fi") && !DL.contains("Fn")) {
7970 const StringRef p3232 = "p:32:32";
7971 size_t Pos = Res.find(p3232);
7972 if (Pos != StringRef::npos)
7973 Res.insert(Pos + p3232.size(), "-Fi8");
7974 }
7975
7976 if (!T.isX86())
7977 return Res;
7978
7979 AddPtr32Ptr64AddrSpaces();
7980
7981 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7982 // for i128 operations prior to this being reflected in the data layout, and
7983 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7984 // boundaries, so although this is a breaking change, the upgrade is expected
7985 // to fix more IR than it breaks.
7986 // Intel MCU is an exception and uses 4-byte-alignment.
7987 if (!T.isOSIAMCU()) {
7988 std::string I128 = "-i128:128";
7989 if (StringRef Ref = Res; !Ref.contains(I128)) {
7990 SmallVector<StringRef, 4> Groups;
7991 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7992 if (R.match(Res, &Groups))
7993 Res = (Groups[1] + I128 + Groups[3]).str();
7994 }
7995 }
7996
7997 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7998 // Raising the alignment is safe because Clang did not produce f80 values in
7999 // the MSVC environment before this upgrade was added.
8000 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8001 StringRef Ref = Res;
8002 auto I = Ref.find("-f80:32-");
8003 if (I != StringRef::npos)
8004 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
8005 }
8006
8007 return Res;
8008}
8009
8010void llvm::UpgradeAttributes(AttrBuilder &B) {
8011 StringRef FramePointer;
8012 Attribute A = B.getAttribute("no-frame-pointer-elim");
8013 if (A.isValid()) {
8014 // The value can be "true" or "false".
8015 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8016 B.removeAttribute("no-frame-pointer-elim");
8017 }
8018 if (B.contains("no-frame-pointer-elim-non-leaf")) {
8019 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8020 if (FramePointer != "all")
8021 FramePointer = "non-leaf";
8022 B.removeAttribute("no-frame-pointer-elim-non-leaf");
8023 }
8024 if (!FramePointer.empty())
8025 B.addAttribute("frame-pointer", FramePointer);
8026
8027 A = B.getAttribute("null-pointer-is-valid");
8028 if (A.isValid()) {
8029 // The value can be "true" or "false".
8030 bool NullPointerIsValid = A.getValueAsString() == "true";
8031 B.removeAttribute("null-pointer-is-valid");
8032 if (NullPointerIsValid)
8033 B.addAttribute(Attribute::NullPointerIsValid);
8034 }
8035
8036 A = B.getAttribute("uniform-work-group-size");
8037 if (A.isValid()) {
8038 StringRef Val = A.getValueAsString();
8039 if (!Val.empty()) {
8040 bool IsTrue = Val == "true";
8041 B.removeAttribute("uniform-work-group-size");
8042 if (IsTrue)
8043 B.addAttribute("uniform-work-group-size");
8044 }
8045 }
8046}
8047
8048void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8049 // clang.arc.attachedcall bundles are now required to have an operand.
8050 // If they don't, it's okay to drop them entirely: when there is an operand,
8051 // the "attachedcall" is meaningful and required, but without an operand,
8052 // it's just a marker NOP. Dropping it merely prevents an optimization.
8053 erase_if(Bundles, [&](OperandBundleDef &OBD) {
8054 return OBD.getTag() == "clang.arc.attachedcall" &&
8055 OBD.inputs().empty();
8056 });
8057}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
Imm
uint64_t Imm
Definition AMDGPUDisassembler.cpp:552
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2689
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1507
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
Definition AutoUpgrade.cpp:1679
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7784
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1561
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6852
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1715
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2529
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1372
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5691
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1339
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1352
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6958
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2790
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3016
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1658
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2764
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:125
upgradeNVVMFPArithCall
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][2])
Definition AutoUpgrade.cpp:3265
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1611
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7605
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1441
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6856
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5551
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7772
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5651
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6819
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:112
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2644
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2901
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1654
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2615
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5188
NVVMFAddIIDs
static constexpr Intrinsic::ID NVVMFAddIIDs[2][2]
Definition AutoUpgrade.cpp:1529
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3002
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7741
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:556
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2941
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1421
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6765
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1568
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7750
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:179
getNVVMFPArithUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][2])
Definition AutoUpgrade.cpp:1537
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2563
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1574
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1219
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1396
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7758
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5258
ConvertModuleFlag
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
Definition AutoUpgrade.cpp:7639
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:96
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5621
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5348
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2880
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5563
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:159
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:794
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5722
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2595
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2627
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1699
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2829
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:67
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:80
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:169
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1194
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2963
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2749
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3278
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1295
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2995
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:3025
NVVMFMulIIDs
static constexpr Intrinsic::ID NVVMFMulIIDs[2][2]
Definition AutoUpgrade.cpp:1532
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5573
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7617
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2911
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:71
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2862
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1323
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1262
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5634
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:143
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7113
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3490
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5669
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:274
T1
#define T1
Definition Mips16ISelLowering.cpp:273
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:480
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:774
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:934
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:786
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:790
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:811
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:850
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:794
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:800
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:854
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:842
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:802
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:822
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:818
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:846
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:981
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:508
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:356
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1595
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1715
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3198
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3481
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1837
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1453
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2490
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2422
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2476
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:2003
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1520
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1684
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:448
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:388
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3621
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2445
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:728
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:565
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2909
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1790
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:703
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1528
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:320
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::op_end
op_iterator op_end() const
Definition Metadata.h:1431
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::op_begin
op_iterator op_begin() const
Definition Metadata.h:1427
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:615
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:597
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1496
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1525
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:107
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1797
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1594
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1586
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1582
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1601
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1893
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1592
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2061
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2095
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:594
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:720
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:425
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1254
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::find
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringRef::consume_front
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:194
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5351
llvm::Value::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Definition Metadata.cpp:1672
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:390
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:549
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:708
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:315
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1451
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:637
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1423
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1416
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5783
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7430
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3256
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:200
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2478
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7821
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:8010
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6656
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6917
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7212
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7386
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7646
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:8048
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6745
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7853
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::UpgradeTBAAStructNode
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
Definition AutoUpgrade.cpp:6701
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2497
llvm::Key
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Definition PassManager.h:695
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:716
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:134
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6723
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getIROptions
const IROptions & getIROptions(const LLVMContext &C)
Definition IROptions.h:27
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6775
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7503
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6675
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6982
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:8065
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:120
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:122
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:121
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
Generated on for LLVM by doxygen 1.14.0