LLVM 23.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/InstVisitor.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsAArch64.h"
35#include "llvm/IR/IntrinsicsAMDGPU.h"
36#include "llvm/IR/IntrinsicsARM.h"
37#include "llvm/IR/IntrinsicsNVPTX.h"
38#include "llvm/IR/IntrinsicsRISCV.h"
39#include "llvm/IR/IntrinsicsWebAssembly.h"
40#include "llvm/IR/IntrinsicsX86.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/Value.h"
46#include "llvm/IR/Verifier.h"
47#include "llvm/Support/AMDGPUAddrSpace.h"
48#include "llvm/Support/CommandLine.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Support/NVPTXAddrSpace.h"
51#include "llvm/Support/NVVMAttributes.h"
52#include "llvm/Support/Regex.h"
53#include "llvm/Support/TimeProfiler.h"
54#include "llvm/TargetParser/Triple.h"
55#include <cstdint>
56#include <cstring>
57#include <numeric>
58
59using namespace llvm;
60
61static cl::opt<bool>
62 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
63 cl::desc("Disable autoupgrade of debug info"));
64
65static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
66
67// Report a fatal error along with the
68// Call Instruction which caused the error
69[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
70 CallBase *CI) {
71 CI->print(llvm::errs());
72 llvm::errs() << "\n";
73 reportFatalUsageError(reason);
74}
75
76// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
77// changed their type from v4f32 to v2i64.
78static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
79 Function *&NewFn) {
80 // Check whether this is an old version of the function, which received
81 // v4f32 arguments.
82 Type *Arg0Type = F->getFunctionType()->getParamType(0);
83 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
84 return false;
85
86 // Yes, it's old, replace it with new version.
87 rename(F);
88 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
89 return true;
90}
91
92// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
93// arguments have changed their type from i32 to i8.
94static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
95 Function *&NewFn) {
96 // Check that the last argument is an i32.
97 Type *LastArgType = F->getFunctionType()->getParamType(
98 F->getFunctionType()->getNumParams() - 1);
99 if (!LastArgType->isIntegerTy(32))
100 return false;
101
102 // Move this function aside and map down.
103 rename(F);
104 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
105 return true;
106}
107
108// Upgrade the declaration of fp compare intrinsics that change return type
109// from scalar to vXi1 mask.
110static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
111 Function *&NewFn) {
112 // Check if the return type is a vector.
113 if (F->getReturnType()->isVectorTy())
114 return false;
115
116 rename(F);
117 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
118 return true;
119}
120
121// Upgrade the declaration of multiply and add bytes intrinsics whose input
122// arguments' types have changed from vectors of i32 to vectors of i8
123static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
124 Function *&NewFn) {
125 // check if input argument type is a vector of i8
126 Type *Arg1Type = F->getFunctionType()->getParamType(1);
127 Type *Arg2Type = F->getFunctionType()->getParamType(2);
128 if (Arg1Type->isVectorTy() &&
129 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
130 Arg2Type->isVectorTy() &&
131 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
132 return false;
133
134 rename(F);
135 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
136 return true;
137}
138
139// Upgrade the declaration of multipy and add words intrinsics whose input
140// arguments' types have changed to vectors of i32 to vectors of i16
141static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
142 Function *&NewFn) {
143 // check if input argument type is a vector of i16
144 Type *Arg1Type = F->getFunctionType()->getParamType(1);
145 Type *Arg2Type = F->getFunctionType()->getParamType(2);
146 if (Arg1Type->isVectorTy() &&
147 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
148 Arg2Type->isVectorTy() &&
149 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
150 return false;
151
152 rename(F);
153 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
154 return true;
155}
156
157static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
158 Function *&NewFn) {
159 if (F->getReturnType()->getScalarType()->isBFloatTy())
160 return false;
161
162 rename(F);
163 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
164 return true;
165}
166
167static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
168 Function *&NewFn) {
169 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
170 return false;
171
172 rename(F);
173 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
174 return true;
175}
176
177static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
178 // All of the intrinsics matches below should be marked with which llvm
179 // version started autoupgrading them. At some point in the future we would
180 // like to use this information to remove upgrade code for some older
181 // intrinsics. It is currently undecided how we will determine that future
182 // point.
183 if (Name.consume_front("avx."))
184 return (Name.starts_with("blend.p") || // Added in 3.7
185 Name == "cvt.ps2.pd.256" || // Added in 3.9
186 Name == "cvtdq2.pd.256" || // Added in 3.9
187 Name == "cvtdq2.ps.256" || // Added in 7.0
188 Name.starts_with("movnt.") || // Added in 3.2
189 Name.starts_with("sqrt.p") || // Added in 7.0
190 Name.starts_with("storeu.") || // Added in 3.9
191 Name.starts_with("vbroadcast.s") || // Added in 3.5
192 Name.starts_with("vbroadcastf128") || // Added in 4.0
193 Name.starts_with("vextractf128.") || // Added in 3.7
194 Name.starts_with("vinsertf128.") || // Added in 3.7
195 Name.starts_with("vperm2f128.") || // Added in 6.0
196 Name.starts_with("vpermil.")); // Added in 3.1
197
198 if (Name.consume_front("avx2."))
199 return (Name == "movntdqa" || // Added in 5.0
200 Name.starts_with("pabs.") || // Added in 6.0
201 Name.starts_with("padds.") || // Added in 8.0
202 Name.starts_with("paddus.") || // Added in 8.0
203 Name.starts_with("pblendd.") || // Added in 3.7
204 Name == "pblendw" || // Added in 3.7
205 Name.starts_with("pbroadcast") || // Added in 3.8
206 Name.starts_with("pcmpeq.") || // Added in 3.1
207 Name.starts_with("pcmpgt.") || // Added in 3.1
208 Name.starts_with("pmax") || // Added in 3.9
209 Name.starts_with("pmin") || // Added in 3.9
210 Name.starts_with("pmovsx") || // Added in 3.9
211 Name.starts_with("pmovzx") || // Added in 3.9
212 Name == "pmul.dq" || // Added in 7.0
213 Name == "pmulu.dq" || // Added in 7.0
214 Name.starts_with("psll.dq") || // Added in 3.7
215 Name.starts_with("psrl.dq") || // Added in 3.7
216 Name.starts_with("psubs.") || // Added in 8.0
217 Name.starts_with("psubus.") || // Added in 8.0
218 Name.starts_with("vbroadcast") || // Added in 3.8
219 Name == "vbroadcasti128" || // Added in 3.7
220 Name == "vextracti128" || // Added in 3.7
221 Name == "vinserti128" || // Added in 3.7
222 Name == "vperm2i128"); // Added in 6.0
223
224 if (Name.consume_front("avx512.")) {
225 if (Name.consume_front("mask."))
226 // 'avx512.mask.*'
227 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
228 Name.starts_with("and.") || // Added in 3.9
229 Name.starts_with("andn.") || // Added in 3.9
230 Name.starts_with("broadcast.s") || // Added in 3.9
231 Name.starts_with("broadcastf32x4.") || // Added in 6.0
232 Name.starts_with("broadcastf32x8.") || // Added in 6.0
233 Name.starts_with("broadcastf64x2.") || // Added in 6.0
234 Name.starts_with("broadcastf64x4.") || // Added in 6.0
235 Name.starts_with("broadcasti32x4.") || // Added in 6.0
236 Name.starts_with("broadcasti32x8.") || // Added in 6.0
237 Name.starts_with("broadcasti64x2.") || // Added in 6.0
238 Name.starts_with("broadcasti64x4.") || // Added in 6.0
239 Name.starts_with("cmp.b") || // Added in 5.0
240 Name.starts_with("cmp.d") || // Added in 5.0
241 Name.starts_with("cmp.q") || // Added in 5.0
242 Name.starts_with("cmp.w") || // Added in 5.0
243 Name.starts_with("compress.b") || // Added in 9.0
244 Name.starts_with("compress.d") || // Added in 9.0
245 Name.starts_with("compress.p") || // Added in 9.0
246 Name.starts_with("compress.q") || // Added in 9.0
247 Name.starts_with("compress.store.") || // Added in 7.0
248 Name.starts_with("compress.w") || // Added in 9.0
249 Name.starts_with("conflict.") || // Added in 9.0
250 Name.starts_with("cvtdq2pd.") || // Added in 4.0
251 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
252 Name == "cvtpd2dq.256" || // Added in 7.0
253 Name == "cvtpd2ps.256" || // Added in 7.0
254 Name == "cvtps2pd.128" || // Added in 7.0
255 Name == "cvtps2pd.256" || // Added in 7.0
256 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
257 Name == "cvtqq2ps.256" || // Added in 9.0
258 Name == "cvtqq2ps.512" || // Added in 9.0
259 Name == "cvttpd2dq.256" || // Added in 7.0
260 Name == "cvttps2dq.128" || // Added in 7.0
261 Name == "cvttps2dq.256" || // Added in 7.0
262 Name.starts_with("cvtudq2pd.") || // Added in 4.0
263 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
264 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
265 Name == "cvtuqq2ps.256" || // Added in 9.0
266 Name == "cvtuqq2ps.512" || // Added in 9.0
267 Name.starts_with("dbpsadbw.") || // Added in 7.0
268 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
269 Name.starts_with("expand.b") || // Added in 9.0
270 Name.starts_with("expand.d") || // Added in 9.0
271 Name.starts_with("expand.load.") || // Added in 7.0
272 Name.starts_with("expand.p") || // Added in 9.0
273 Name.starts_with("expand.q") || // Added in 9.0
274 Name.starts_with("expand.w") || // Added in 9.0
275 Name.starts_with("fpclass.p") || // Added in 7.0
276 Name.starts_with("insert") || // Added in 4.0
277 Name.starts_with("load.") || // Added in 3.9
278 Name.starts_with("loadu.") || // Added in 3.9
279 Name.starts_with("lzcnt.") || // Added in 5.0
280 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
281 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
282 Name.starts_with("movddup") || // Added in 3.9
283 Name.starts_with("move.s") || // Added in 4.0
284 Name.starts_with("movshdup") || // Added in 3.9
285 Name.starts_with("movsldup") || // Added in 3.9
286 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
287 Name.starts_with("or.") || // Added in 3.9
288 Name.starts_with("pabs.") || // Added in 6.0
289 Name.starts_with("packssdw.") || // Added in 5.0
290 Name.starts_with("packsswb.") || // Added in 5.0
291 Name.starts_with("packusdw.") || // Added in 5.0
292 Name.starts_with("packuswb.") || // Added in 5.0
293 Name.starts_with("padd.") || // Added in 4.0
294 Name.starts_with("padds.") || // Added in 8.0
295 Name.starts_with("paddus.") || // Added in 8.0
296 Name.starts_with("palignr.") || // Added in 3.9
297 Name.starts_with("pand.") || // Added in 3.9
298 Name.starts_with("pandn.") || // Added in 3.9
299 Name.starts_with("pavg") || // Added in 6.0
300 Name.starts_with("pbroadcast") || // Added in 6.0
301 Name.starts_with("pcmpeq.") || // Added in 3.9
302 Name.starts_with("pcmpgt.") || // Added in 3.9
303 Name.starts_with("perm.df.") || // Added in 3.9
304 Name.starts_with("perm.di.") || // Added in 3.9
305 Name.starts_with("permvar.") || // Added in 7.0
306 Name.starts_with("pmaddubs.w.") || // Added in 7.0
307 Name.starts_with("pmaddw.d.") || // Added in 7.0
308 Name.starts_with("pmax") || // Added in 4.0
309 Name.starts_with("pmin") || // Added in 4.0
310 Name == "pmov.qd.256" || // Added in 9.0
311 Name == "pmov.qd.512" || // Added in 9.0
312 Name == "pmov.wb.256" || // Added in 9.0
313 Name == "pmov.wb.512" || // Added in 9.0
314 Name.starts_with("pmovsx") || // Added in 4.0
315 Name.starts_with("pmovzx") || // Added in 4.0
316 Name.starts_with("pmul.dq.") || // Added in 4.0
317 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
318 Name.starts_with("pmulh.w.") || // Added in 7.0
319 Name.starts_with("pmulhu.w.") || // Added in 7.0
320 Name.starts_with("pmull.") || // Added in 4.0
321 Name.starts_with("pmultishift.qb.") || // Added in 8.0
322 Name.starts_with("pmulu.dq.") || // Added in 4.0
323 Name.starts_with("por.") || // Added in 3.9
324 Name.starts_with("prol.") || // Added in 8.0
325 Name.starts_with("prolv.") || // Added in 8.0
326 Name.starts_with("pror.") || // Added in 8.0
327 Name.starts_with("prorv.") || // Added in 8.0
328 Name.starts_with("pshuf.b.") || // Added in 4.0
329 Name.starts_with("pshuf.d.") || // Added in 3.9
330 Name.starts_with("pshufh.w.") || // Added in 3.9
331 Name.starts_with("pshufl.w.") || // Added in 3.9
332 Name.starts_with("psll.d") || // Added in 4.0
333 Name.starts_with("psll.q") || // Added in 4.0
334 Name.starts_with("psll.w") || // Added in 4.0
335 Name.starts_with("pslli") || // Added in 4.0
336 Name.starts_with("psllv") || // Added in 4.0
337 Name.starts_with("psra.d") || // Added in 4.0
338 Name.starts_with("psra.q") || // Added in 4.0
339 Name.starts_with("psra.w") || // Added in 4.0
340 Name.starts_with("psrai") || // Added in 4.0
341 Name.starts_with("psrav") || // Added in 4.0
342 Name.starts_with("psrl.d") || // Added in 4.0
343 Name.starts_with("psrl.q") || // Added in 4.0
344 Name.starts_with("psrl.w") || // Added in 4.0
345 Name.starts_with("psrli") || // Added in 4.0
346 Name.starts_with("psrlv") || // Added in 4.0
347 Name.starts_with("psub.") || // Added in 4.0
348 Name.starts_with("psubs.") || // Added in 8.0
349 Name.starts_with("psubus.") || // Added in 8.0
350 Name.starts_with("pternlog.") || // Added in 7.0
351 Name.starts_with("punpckh") || // Added in 3.9
352 Name.starts_with("punpckl") || // Added in 3.9
353 Name.starts_with("pxor.") || // Added in 3.9
354 Name.starts_with("shuf.f") || // Added in 6.0
355 Name.starts_with("shuf.i") || // Added in 6.0
356 Name.starts_with("shuf.p") || // Added in 4.0
357 Name.starts_with("sqrt.p") || // Added in 7.0
358 Name.starts_with("store.b.") || // Added in 3.9
359 Name.starts_with("store.d.") || // Added in 3.9
360 Name.starts_with("store.p") || // Added in 3.9
361 Name.starts_with("store.q.") || // Added in 3.9
362 Name.starts_with("store.w.") || // Added in 3.9
363 Name == "store.ss" || // Added in 7.0
364 Name.starts_with("storeu.") || // Added in 3.9
365 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
366 Name.starts_with("ucmp.") || // Added in 5.0
367 Name.starts_with("unpckh.") || // Added in 3.9
368 Name.starts_with("unpckl.") || // Added in 3.9
369 Name.starts_with("valign.") || // Added in 4.0
370 Name == "vcvtph2ps.128" || // Added in 11.0
371 Name == "vcvtph2ps.256" || // Added in 11.0
372 Name.starts_with("vextract") || // Added in 4.0
373 Name.starts_with("vfmadd.") || // Added in 7.0
374 Name.starts_with("vfmaddsub.") || // Added in 7.0
375 Name.starts_with("vfnmadd.") || // Added in 7.0
376 Name.starts_with("vfnmsub.") || // Added in 7.0
377 Name.starts_with("vpdpbusd.") || // Added in 7.0
378 Name.starts_with("vpdpbusds.") || // Added in 7.0
379 Name.starts_with("vpdpwssd.") || // Added in 7.0
380 Name.starts_with("vpdpwssds.") || // Added in 7.0
381 Name.starts_with("vpermi2var.") || // Added in 7.0
382 Name.starts_with("vpermil.p") || // Added in 3.9
383 Name.starts_with("vpermilvar.") || // Added in 4.0
384 Name.starts_with("vpermt2var.") || // Added in 7.0
385 Name.starts_with("vpmadd52") || // Added in 7.0
386 Name.starts_with("vpshld.") || // Added in 7.0
387 Name.starts_with("vpshldv.") || // Added in 8.0
388 Name.starts_with("vpshrd.") || // Added in 7.0
389 Name.starts_with("vpshrdv.") || // Added in 8.0
390 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
391 Name.starts_with("xor.")); // Added in 3.9
392
393 if (Name.consume_front("mask3."))
394 // 'avx512.mask3.*'
395 return (Name.starts_with("vfmadd.") || // Added in 7.0
396 Name.starts_with("vfmaddsub.") || // Added in 7.0
397 Name.starts_with("vfmsub.") || // Added in 7.0
398 Name.starts_with("vfmsubadd.") || // Added in 7.0
399 Name.starts_with("vfnmsub.")); // Added in 7.0
400
401 if (Name.consume_front("maskz."))
402 // 'avx512.maskz.*'
403 return (Name.starts_with("pternlog.") || // Added in 7.0
404 Name.starts_with("vfmadd.") || // Added in 7.0
405 Name.starts_with("vfmaddsub.") || // Added in 7.0
406 Name.starts_with("vpdpbusd.") || // Added in 7.0
407 Name.starts_with("vpdpbusds.") || // Added in 7.0
408 Name.starts_with("vpdpwssd.") || // Added in 7.0
409 Name.starts_with("vpdpwssds.") || // Added in 7.0
410 Name.starts_with("vpermt2var.") || // Added in 7.0
411 Name.starts_with("vpmadd52") || // Added in 7.0
412 Name.starts_with("vpshldv.") || // Added in 8.0
413 Name.starts_with("vpshrdv.")); // Added in 8.0
414
415 // 'avx512.*'
416 return (Name == "movntdqa" || // Added in 5.0
417 Name == "pmul.dq.512" || // Added in 7.0
418 Name == "pmulu.dq.512" || // Added in 7.0
419 Name.starts_with("broadcastm") || // Added in 6.0
420 Name.starts_with("cmp.p") || // Added in 12.0
421 Name.starts_with("cvtb2mask.") || // Added in 7.0
422 Name.starts_with("cvtd2mask.") || // Added in 7.0
423 Name.starts_with("cvtmask2") || // Added in 5.0
424 Name.starts_with("cvtq2mask.") || // Added in 7.0
425 Name == "cvtusi2sd" || // Added in 7.0
426 Name.starts_with("cvtw2mask.") || // Added in 7.0
427 Name == "kand.w" || // Added in 7.0
428 Name == "kandn.w" || // Added in 7.0
429 Name == "knot.w" || // Added in 7.0
430 Name == "kor.w" || // Added in 7.0
431 Name == "kortestc.w" || // Added in 7.0
432 Name == "kortestz.w" || // Added in 7.0
433 Name.starts_with("kunpck") || // added in 6.0
434 Name == "kxnor.w" || // Added in 7.0
435 Name == "kxor.w" || // Added in 7.0
436 Name.starts_with("padds.") || // Added in 8.0
437 Name.starts_with("pbroadcast") || // Added in 3.9
438 Name.starts_with("prol") || // Added in 8.0
439 Name.starts_with("pror") || // Added in 8.0
440 Name.starts_with("psll.dq") || // Added in 3.9
441 Name.starts_with("psrl.dq") || // Added in 3.9
442 Name.starts_with("psubs.") || // Added in 8.0
443 Name.starts_with("ptestm") || // Added in 6.0
444 Name.starts_with("ptestnm") || // Added in 6.0
445 Name.starts_with("storent.") || // Added in 3.9
446 Name.starts_with("vbroadcast.s") || // Added in 7.0
447 Name.starts_with("vpshld.") || // Added in 8.0
448 Name.starts_with("vpshrd.")); // Added in 8.0
449 }
450
451 if (Name.consume_front("fma."))
452 return (Name.starts_with("vfmadd.") || // Added in 7.0
453 Name.starts_with("vfmsub.") || // Added in 7.0
454 Name.starts_with("vfmsubadd.") || // Added in 7.0
455 Name.starts_with("vfnmadd.") || // Added in 7.0
456 Name.starts_with("vfnmsub.")); // Added in 7.0
457
458 if (Name.consume_front("fma4."))
459 return Name.starts_with("vfmadd.s"); // Added in 7.0
460
461 if (Name.consume_front("sse."))
462 return (Name == "add.ss" || // Added in 4.0
463 Name == "cvtsi2ss" || // Added in 7.0
464 Name == "cvtsi642ss" || // Added in 7.0
465 Name == "div.ss" || // Added in 4.0
466 Name == "mul.ss" || // Added in 4.0
467 Name.starts_with("sqrt.p") || // Added in 7.0
468 Name == "sqrt.ss" || // Added in 7.0
469 Name.starts_with("storeu.") || // Added in 3.9
470 Name == "sub.ss"); // Added in 4.0
471
472 if (Name.consume_front("sse2."))
473 return (Name == "add.sd" || // Added in 4.0
474 Name == "cvtdq2pd" || // Added in 3.9
475 Name == "cvtdq2ps" || // Added in 7.0
476 Name == "cvtps2pd" || // Added in 3.9
477 Name == "cvtsi2sd" || // Added in 7.0
478 Name == "cvtsi642sd" || // Added in 7.0
479 Name == "cvtss2sd" || // Added in 7.0
480 Name == "div.sd" || // Added in 4.0
481 Name == "mul.sd" || // Added in 4.0
482 Name.starts_with("padds.") || // Added in 8.0
483 Name.starts_with("paddus.") || // Added in 8.0
484 Name.starts_with("pcmpeq.") || // Added in 3.1
485 Name.starts_with("pcmpgt.") || // Added in 3.1
486 Name == "pmaxs.w" || // Added in 3.9
487 Name == "pmaxu.b" || // Added in 3.9
488 Name == "pmins.w" || // Added in 3.9
489 Name == "pminu.b" || // Added in 3.9
490 Name == "pmulu.dq" || // Added in 7.0
491 Name.starts_with("pshuf") || // Added in 3.9
492 Name.starts_with("psll.dq") || // Added in 3.7
493 Name.starts_with("psrl.dq") || // Added in 3.7
494 Name.starts_with("psubs.") || // Added in 8.0
495 Name.starts_with("psubus.") || // Added in 8.0
496 Name.starts_with("sqrt.p") || // Added in 7.0
497 Name == "sqrt.sd" || // Added in 7.0
498 Name == "storel.dq" || // Added in 3.9
499 Name.starts_with("storeu.") || // Added in 3.9
500 Name == "sub.sd"); // Added in 4.0
501
502 if (Name.consume_front("sse41."))
503 return (Name.starts_with("blendp") || // Added in 3.7
504 Name == "movntdqa" || // Added in 5.0
505 Name == "pblendw" || // Added in 3.7
506 Name == "pmaxsb" || // Added in 3.9
507 Name == "pmaxsd" || // Added in 3.9
508 Name == "pmaxud" || // Added in 3.9
509 Name == "pmaxuw" || // Added in 3.9
510 Name == "pminsb" || // Added in 3.9
511 Name == "pminsd" || // Added in 3.9
512 Name == "pminud" || // Added in 3.9
513 Name == "pminuw" || // Added in 3.9
514 Name.starts_with("pmovsx") || // Added in 3.8
515 Name.starts_with("pmovzx") || // Added in 3.9
516 Name == "pmuldq"); // Added in 7.0
517
518 if (Name.consume_front("sse42."))
519 return Name == "crc32.64.8"; // Added in 3.4
520
521 if (Name.consume_front("sse4a."))
522 return Name.starts_with("movnt."); // Added in 3.9
523
524 if (Name.consume_front("ssse3."))
525 return (Name == "pabs.b.128" || // Added in 6.0
526 Name == "pabs.d.128" || // Added in 6.0
527 Name == "pabs.w.128"); // Added in 6.0
528
529 if (Name.consume_front("xop."))
530 return (Name == "vpcmov" || // Added in 3.8
531 Name == "vpcmov.256" || // Added in 5.0
532 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
533 Name.starts_with("vprot")); // Added in 8.0
534
535 return (Name == "addcarry.u32" || // Added in 8.0
536 Name == "addcarry.u64" || // Added in 8.0
537 Name == "addcarryx.u32" || // Added in 8.0
538 Name == "addcarryx.u64" || // Added in 8.0
539 Name == "subborrow.u32" || // Added in 8.0
540 Name == "subborrow.u64" || // Added in 8.0
541 Name.starts_with("vcvtph2ps.")); // Added in 11.0
542}
543
544static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
545 Function *&NewFn) {
546 // Only handle intrinsics that start with "x86.".
547 if (!Name.consume_front("x86."))
548 return false;
549
550 if (shouldUpgradeX86Intrinsic(F, Name)) {
551 NewFn = nullptr;
552 return true;
553 }
554
555 if (Name == "rdtscp") { // Added in 8.0
556 // If this intrinsic has 0 operands, it's the new version.
557 if (F->getFunctionType()->getNumParams() == 0)
558 return false;
559
560 rename(F);
561 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
562 Intrinsic::x86_rdtscp);
563 return true;
564 }
565
566 Intrinsic::ID ID;
567
568 // SSE4.1 ptest functions may have an old signature.
569 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
570 ID = StringSwitch<Intrinsic::ID>(Name)
571 .Case("c", Intrinsic::x86_sse41_ptestc)
572 .Case("z", Intrinsic::x86_sse41_ptestz)
573 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
574 .Default(Intrinsic::not_intrinsic);
575 if (ID != Intrinsic::not_intrinsic)
576 return upgradePTESTIntrinsic(F, ID, NewFn);
577
578 return false;
579 }
580
581 // Several blend and other instructions with masks used the wrong number of
582 // bits.
583
584 // Added in 3.6
585 ID = StringSwitch<Intrinsic::ID>(Name)
586 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
587 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
588 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
589 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
590 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
591 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
592 .Default(Intrinsic::not_intrinsic);
593 if (ID != Intrinsic::not_intrinsic)
594 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
595
596 if (Name.consume_front("avx512.")) {
597 if (Name.consume_front("mask.cmp.")) {
598 // Added in 7.0
599 ID = StringSwitch<Intrinsic::ID>(Name)
600 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
601 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
602 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
603 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
604 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
605 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
606 .Default(Intrinsic::not_intrinsic);
607 if (ID != Intrinsic::not_intrinsic)
608 return upgradeX86MaskedFPCompare(F, ID, NewFn);
609 } else if (Name.starts_with("vpdpbusd.") ||
610 Name.starts_with("vpdpbusds.")) {
611 // Added in 21.1
612 ID = StringSwitch<Intrinsic::ID>(Name)
613 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
614 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
615 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
616 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
617 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
618 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
619 .Default(Intrinsic::not_intrinsic);
620 if (ID != Intrinsic::not_intrinsic)
621 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
622 } else if (Name.starts_with("vpdpwssd.") ||
623 Name.starts_with("vpdpwssds.")) {
624 // Added in 21.1
625 ID = StringSwitch<Intrinsic::ID>(Name)
626 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
627 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
628 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
629 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
630 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
631 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
632 .Default(Intrinsic::not_intrinsic);
633 if (ID != Intrinsic::not_intrinsic)
634 return upgradeX86MultiplyAddWords(F, ID, NewFn);
635 }
636 return false; // No other 'x86.avx512.*'.
637 }
638
639 if (Name.consume_front("avx2.")) {
640 if (Name.consume_front("vpdpb")) {
641 // Added in 21.1
642 ID = StringSwitch<Intrinsic::ID>(Name)
643 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
644 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
645 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
646 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
647 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
648 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
649 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
650 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
651 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
652 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
653 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
654 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
655 .Default(Intrinsic::not_intrinsic);
656 if (ID != Intrinsic::not_intrinsic)
657 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
658 } else if (Name.consume_front("vpdpw")) {
659 // Added in 21.1
660 ID = StringSwitch<Intrinsic::ID>(Name)
661 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
662 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
663 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
664 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
665 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
666 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
667 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
668 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
669 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
670 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
671 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
672 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
673 .Default(Intrinsic::not_intrinsic);
674 if (ID != Intrinsic::not_intrinsic)
675 return upgradeX86MultiplyAddWords(F, ID, NewFn);
676 }
677 return false; // No other 'x86.avx2.*'
678 }
679
680 if (Name.consume_front("avx10.")) {
681 if (Name.consume_front("vpdpb")) {
682 // Added in 21.1
683 ID = StringSwitch<Intrinsic::ID>(Name)
684 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
685 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
686 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
687 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
688 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
689 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
690 .Default(Intrinsic::not_intrinsic);
691 if (ID != Intrinsic::not_intrinsic)
692 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
693 } else if (Name.consume_front("vpdpw")) {
694 ID = StringSwitch<Intrinsic::ID>(Name)
695 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
696 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
697 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
698 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
699 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
700 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
701 .Default(Intrinsic::not_intrinsic);
702 if (ID != Intrinsic::not_intrinsic)
703 return upgradeX86MultiplyAddWords(F, ID, NewFn);
704 }
705 return false; // No other 'x86.avx10.*'
706 }
707
708 if (Name.consume_front("avx512bf16.")) {
709 // Added in 9.0
710 ID = StringSwitch<Intrinsic::ID>(Name)
711 .Case("cvtne2ps2bf16.128",
712 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
713 .Case("cvtne2ps2bf16.256",
714 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
715 .Case("cvtne2ps2bf16.512",
716 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
717 .Case("mask.cvtneps2bf16.128",
718 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
719 .Case("cvtneps2bf16.256",
720 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
721 .Case("cvtneps2bf16.512",
722 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
723 .Default(Intrinsic::not_intrinsic);
724 if (ID != Intrinsic::not_intrinsic)
725 return upgradeX86BF16Intrinsic(F, ID, NewFn);
726
727 // Added in 9.0
728 ID = StringSwitch<Intrinsic::ID>(Name)
729 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
730 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
731 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
732 .Default(Intrinsic::not_intrinsic);
733 if (ID != Intrinsic::not_intrinsic)
734 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
735 return false; // No other 'x86.avx512bf16.*'.
736 }
737
738 if (Name.consume_front("xop.")) {
739 Intrinsic::ID ID = Intrinsic::not_intrinsic;
740 if (Name.starts_with("vpermil2")) { // Added in 3.9
741 // Upgrade any XOP PERMIL2 index operand still using a float/double
742 // vector.
743 auto Idx = F->getFunctionType()->getParamType(2);
744 if (Idx->isFPOrFPVectorTy()) {
745 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
746 unsigned EltSize = Idx->getScalarSizeInBits();
747 if (EltSize == 64 && IdxSize == 128)
748 ID = Intrinsic::x86_xop_vpermil2pd;
749 else if (EltSize == 32 && IdxSize == 128)
750 ID = Intrinsic::x86_xop_vpermil2ps;
751 else if (EltSize == 64 && IdxSize == 256)
752 ID = Intrinsic::x86_xop_vpermil2pd_256;
753 else
754 ID = Intrinsic::x86_xop_vpermil2ps_256;
755 }
756 } else if (F->arg_size() == 2)
757 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
758 ID = StringSwitch<Intrinsic::ID>(Name)
759 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
760 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
761 .Default(Intrinsic::not_intrinsic);
762
763 if (ID != Intrinsic::not_intrinsic) {
764 rename(F);
765 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
766 return true;
767 }
768 return false; // No other 'x86.xop.*'
769 }
770
771 if (Name == "seh.recoverfp") {
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
773 Intrinsic::eh_recoverfp);
774 return true;
775 }
776
777 return false;
778}
779
780// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
781// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
782static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
783 StringRef Name,
784 Function *&NewFn) {
785 if (Name.starts_with("rbit")) {
786 // '(arm|aarch64).rbit'.
787 NewFn = Intrinsic::getOrInsertDeclaration(
788 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
789 return true;
790 }
791
792 if (Name == "thread.pointer") {
793 // '(arm|aarch64).thread.pointer'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
796 return true;
797 }
798
799 bool Neon = Name.consume_front("neon.");
800 if (Neon) {
801 // '(arm|aarch64).neon.*'.
802 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
803 // v16i8 respectively.
804 if (Name.consume_front("bfdot.")) {
805 // (arm|aarch64).neon.bfdot.*'.
806 Intrinsic::ID ID =
807 StringSwitch<Intrinsic::ID>(Name)
808 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
809 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
810 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
811 .Default(Intrinsic::not_intrinsic);
812 if (ID != Intrinsic::not_intrinsic) {
813 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
814 assert((OperandWidth == 64 || OperandWidth == 128) &&
815 "Unexpected operand width");
816 LLVMContext &Ctx = F->getParent()->getContext();
817 std::array<Type *, 2> Tys{
818 {F->getReturnType(),
819 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
820 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
821 return true;
822 }
823 return false; // No other '(arm|aarch64).neon.bfdot.*'.
824 }
825
826 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
827 // anymore and accept v8bf16 instead of v16i8.
828 if (Name.consume_front("bfm")) {
829 // (arm|aarch64).neon.bfm*'.
830 if (Name.consume_back(".v4f32.v16i8")) {
831 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
832 Intrinsic::ID ID =
833 StringSwitch<Intrinsic::ID>(Name)
834 .Case("mla",
835 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
836 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
837 .Case("lalb",
838 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
839 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
840 .Case("lalt",
841 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
842 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
843 .Default(Intrinsic::not_intrinsic);
844 if (ID != Intrinsic::not_intrinsic) {
845 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
846 return true;
847 }
848 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
849 }
850 return false; // No other '(arm|aarch64).neon.bfm*.
851 }
852 // Continue on to Aarch64 Neon or Arm Neon.
853 }
854 // Continue on to Arm or Aarch64.
855
856 if (IsArm) {
857 // 'arm.*'.
858 if (Neon) {
859 // 'arm.neon.*'.
860 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
861 .StartsWith("vclz.", Intrinsic::ctlz)
862 .StartsWith("vcnt.", Intrinsic::ctpop)
863 .StartsWith("vqadds.", Intrinsic::sadd_sat)
864 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
865 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
866 .StartsWith("vqsubu.", Intrinsic::usub_sat)
867 .StartsWith("vrinta.", Intrinsic::round)
868 .StartsWith("vrintn.", Intrinsic::roundeven)
869 .StartsWith("vrintm.", Intrinsic::floor)
870 .StartsWith("vrintp.", Intrinsic::ceil)
871 .StartsWith("vrintx.", Intrinsic::rint)
872 .StartsWith("vrintz.", Intrinsic::trunc)
873 .Default(Intrinsic::not_intrinsic);
874 if (ID != Intrinsic::not_intrinsic) {
875 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
876 F->arg_begin()->getType());
877 return true;
878 }
879
880 if (Name.consume_front("vst")) {
881 // 'arm.neon.vst*'.
882 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
883 SmallVector<StringRef, 2> Groups;
884 if (vstRegex.match(Name, &Groups)) {
885 static const Intrinsic::ID StoreInts[] = {
886 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
887 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
888
889 static const Intrinsic::ID StoreLaneInts[] = {
890 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
891 Intrinsic::arm_neon_vst4lane};
892
893 auto fArgs = F->getFunctionType()->params();
894 Type *Tys[] = {fArgs[0], fArgs[1]};
895 if (Groups[1].size() == 1)
896 NewFn = Intrinsic::getOrInsertDeclaration(
897 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
898 else
899 NewFn = Intrinsic::getOrInsertDeclaration(
900 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
901 return true;
902 }
903 return false; // No other 'arm.neon.vst*'.
904 }
905
906 return false; // No other 'arm.neon.*'.
907 }
908
909 if (Name.consume_front("mve.")) {
910 // 'arm.mve.*'.
911 if (Name == "vctp64") {
912 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
913 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
914 // the function and deal with it below in UpgradeIntrinsicCall.
915 rename(F);
916 return true;
917 }
918 return false; // Not 'arm.mve.vctp64'.
919 }
920
921 if (Name.starts_with("vrintn.v")) {
922 NewFn = Intrinsic::getOrInsertDeclaration(
923 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
924 return true;
925 }
926
927 // These too are changed to accept a v2i1 instead of the old v4i1.
928 if (Name.consume_back(".v4i1")) {
929 // 'arm.mve.*.v4i1'.
930 if (Name.consume_back(".predicated.v2i64.v4i32"))
931 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
932 return Name == "mull.int" || Name == "vqdmull";
933
934 if (Name.consume_back(".v2i64")) {
935 // 'arm.mve.*.v2i64.v4i1'
936 bool IsGather = Name.consume_front("vldr.gather.");
937 if (IsGather || Name.consume_front("vstr.scatter.")) {
938 if (Name.consume_front("base.")) {
939 // Optional 'wb.' prefix.
940 Name.consume_front("wb.");
941 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
942 // predicated.v2i64.v2i64.v4i1'.
943 return Name == "predicated.v2i64";
944 }
945
946 if (Name.consume_front("offset.predicated."))
947 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
948 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
949
950 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
951 return false;
952 }
953
954 return false; // No other 'arm.mve.*.v2i64.v4i1'.
955 }
956 return false; // No other 'arm.mve.*.v4i1'.
957 }
958 return false; // No other 'arm.mve.*'.
959 }
960
961 if (Name.consume_front("cde.vcx")) {
962 // 'arm.cde.vcx*'.
963 if (Name.consume_back(".predicated.v2i64.v4i1"))
964 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
965 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
966 Name == "3q" || Name == "3qa";
967
968 return false; // No other 'arm.cde.vcx*'.
969 }
970 } else {
971 // 'aarch64.*'.
972 if (Neon) {
973 // 'aarch64.neon.*'.
974 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
975 .StartsWith("frintn", Intrinsic::roundeven)
976 .StartsWith("rbit", Intrinsic::bitreverse)
977 .Default(Intrinsic::not_intrinsic);
978 if (ID != Intrinsic::not_intrinsic) {
979 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
980 F->arg_begin()->getType());
981 return true;
982 }
983
984 if (Name.starts_with("addp")) {
985 // 'aarch64.neon.addp*'.
986 if (F->arg_size() != 2)
987 return false; // Invalid IR.
988 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
989 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
990 NewFn = Intrinsic::getOrInsertDeclaration(
991 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
992 return true;
993 }
994 }
995
996 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
997 if (Name.starts_with("bfcvt")) {
998 NewFn = nullptr;
999 return true;
1000 }
1001
1002 return false; // No other 'aarch64.neon.*'.
1003 }
1004 if (Name.consume_front("sve.")) {
1005 // 'aarch64.sve.*'.
1006 if (Name.consume_front("bf")) {
1007 if (Name == "mmla") {
1008 Type *Tys[] = {F->getReturnType(),
1009 std::next(F->arg_begin())->getType()};
1010 NewFn = Intrinsic::getOrInsertDeclaration(
1011 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1012 return true;
1013 }
1014 if (Name.consume_back(".lane")) {
1015 // 'aarch64.sve.bf*.lane'.
1016 Intrinsic::ID ID =
1017 StringSwitch<Intrinsic::ID>(Name)
1018 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1019 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1020 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1021 .Default(Intrinsic::not_intrinsic);
1022 if (ID != Intrinsic::not_intrinsic) {
1023 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1024 return true;
1025 }
1026 return false; // No other 'aarch64.sve.bf*.lane'.
1027 }
1028 return false; // No other 'aarch64.sve.bf*'.
1029 }
1030
1031 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1032 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1033 NewFn = nullptr;
1034 return true;
1035 }
1036
1037 if (Name.consume_front("addqv")) {
1038 // 'aarch64.sve.addqv'.
1039 if (!F->getReturnType()->isFPOrFPVectorTy())
1040 return false;
1041
1042 auto Args = F->getFunctionType()->params();
1043 Type *Tys[] = {F->getReturnType(), Args[1]};
1044 NewFn = Intrinsic::getOrInsertDeclaration(
1045 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1046 return true;
1047 }
1048
1049 if (Name.consume_front("ld")) {
1050 // 'aarch64.sve.ld*'.
1051 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1052 if (LdRegex.match(Name)) {
1053 Type *ScalarTy =
1054 cast<VectorType>(F->getReturnType())->getElementType();
1055 ElementCount EC =
1056 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1057 assert(F->arg_size() == 2 &&
1058 "Expected 2 arguments for ld* intrinsic.");
1059 Type *PtrTy = F->getArg(1)->getType();
1060 Type *Ty = VectorType::get(ScalarTy, EC);
1061 static const Intrinsic::ID LoadIDs[] = {
1062 Intrinsic::aarch64_sve_ld2_sret,
1063 Intrinsic::aarch64_sve_ld3_sret,
1064 Intrinsic::aarch64_sve_ld4_sret,
1065 };
1066 NewFn = Intrinsic::getOrInsertDeclaration(
1067 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1068 return true;
1069 }
1070 return false; // No other 'aarch64.sve.ld*'.
1071 }
1072
1073 if (Name.consume_front("tuple.")) {
1074 // 'aarch64.sve.tuple.*'.
1075 if (Name.starts_with("get")) {
1076 // 'aarch64.sve.tuple.get*'.
1077 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1078 NewFn = Intrinsic::getOrInsertDeclaration(
1079 F->getParent(), Intrinsic::vector_extract, Tys);
1080 return true;
1081 }
1082
1083 if (Name.starts_with("set")) {
1084 // 'aarch64.sve.tuple.set*'.
1085 auto Args = F->getFunctionType()->params();
1086 Type *Tys[] = {Args[0], Args[2], Args[1]};
1087 NewFn = Intrinsic::getOrInsertDeclaration(
1088 F->getParent(), Intrinsic::vector_insert, Tys);
1089 return true;
1090 }
1091
1092 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1093 if (CreateTupleRegex.match(Name)) {
1094 // 'aarch64.sve.tuple.create*'.
1095 auto Args = F->getFunctionType()->params();
1096 Type *Tys[] = {F->getReturnType(), Args[1]};
1097 NewFn = Intrinsic::getOrInsertDeclaration(
1098 F->getParent(), Intrinsic::vector_insert, Tys);
1099 return true;
1100 }
1101 return false; // No other 'aarch64.sve.tuple.*'.
1102 }
1103
1104 if (Name.starts_with("rev.nxv")) {
1105 // 'aarch64.sve.rev.<Ty>'
1106 NewFn = Intrinsic::getOrInsertDeclaration(
1107 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1108 return true;
1109 }
1110
1111 return false; // No other 'aarch64.sve.*'.
1112 }
1113 }
1114 return false; // No other 'arm.*', 'aarch64.*'.
1115}
1116
1117static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1118 StringRef Name) {
1119 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1120 Intrinsic::ID ID =
1121 StringSwitch<Intrinsic::ID>(Name)
1122 .Case("im2col.3d",
1123 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1124 .Case("im2col.4d",
1125 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1126 .Case("im2col.5d",
1127 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1128 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1129 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1130 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1131 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1132 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1133 .Default(Intrinsic::not_intrinsic);
1134
1135 if (ID == Intrinsic::not_intrinsic)
1136 return ID;
1137
1138 // These intrinsics may need upgrade for two reasons:
1139 // (1) When the address-space of the first argument is shared[AS=3]
1140 // (and we upgrade it to use shared_cluster address-space[AS=7])
1141 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1142 NVPTXAS::ADDRESS_SPACE_SHARED)
1143 return ID;
1144
1145 // (2) When there are only two boolean flag arguments at the end:
1146 //
1147 // The last three parameters of the older version of these
1148 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1149 //
1150 // The newer version reads as:
1151 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1152 //
1153 // So, when the type of the [N-3]rd argument is "not i1", then
1154 // it is the older version and we need to upgrade.
1155 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1156 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1157 if (!ArgType->isIntegerTy(1))
1158 return ID;
1159 }
1160
1161 return Intrinsic::not_intrinsic;
1162}
1163
1164static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1165 StringRef Name) {
1166 if (Name.consume_front("mapa.shared.cluster"))
1167 if (F->getReturnType()->getPointerAddressSpace() ==
1168 NVPTXAS::ADDRESS_SPACE_SHARED)
1169 return Intrinsic::nvvm_mapa_shared_cluster;
1170
1171 if (Name.consume_front("cp.async.bulk.")) {
1172 Intrinsic::ID ID =
1173 StringSwitch<Intrinsic::ID>(Name)
1174 .Case("global.to.shared.cluster",
1175 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1176 .Case("shared.cta.to.cluster",
1177 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1178 .Default(Intrinsic::not_intrinsic);
1179
1180 if (ID != Intrinsic::not_intrinsic)
1181 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1182 NVPTXAS::ADDRESS_SPACE_SHARED)
1183 return ID;
1184 }
1185
1186 return Intrinsic::not_intrinsic;
1187}
1188
1189static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1190 if (Name.consume_front("fma.rn."))
1191 return StringSwitch<Intrinsic::ID>(Name)
1192 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1193 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1194 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1195 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1196 .Default(Intrinsic::not_intrinsic);
1197
1198 if (Name.consume_front("fmax."))
1199 return StringSwitch<Intrinsic::ID>(Name)
1200 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1201 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1202 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1203 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1204 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1205 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1206 .Case("ftz.nan.xorsign.abs.bf16",
1207 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1208 .Case("ftz.nan.xorsign.abs.bf16x2",
1209 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1210 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1211 .Case("ftz.xorsign.abs.bf16x2",
1212 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1213 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1214 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1215 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1216 .Case("nan.xorsign.abs.bf16x2",
1217 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1218 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1219 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1220 .Default(Intrinsic::not_intrinsic);
1221
1222 if (Name.consume_front("fmin."))
1223 return StringSwitch<Intrinsic::ID>(Name)
1224 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1225 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1226 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1227 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1228 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1229 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1230 .Case("ftz.nan.xorsign.abs.bf16",
1231 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1232 .Case("ftz.nan.xorsign.abs.bf16x2",
1233 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1234 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1235 .Case("ftz.xorsign.abs.bf16x2",
1236 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1237 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1238 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1239 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1240 .Case("nan.xorsign.abs.bf16x2",
1241 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1242 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1243 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1244 .Default(Intrinsic::not_intrinsic);
1245
1246 if (Name.consume_front("neg."))
1247 return StringSwitch<Intrinsic::ID>(Name)
1248 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1249 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1250 .Default(Intrinsic::not_intrinsic);
1251
1252 return Intrinsic::not_intrinsic;
1253}
1254
1255static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1256 return Name.consume_front("local") || Name.consume_front("shared") ||
1257 Name.consume_front("global") || Name.consume_front("constant") ||
1258 Name.consume_front("param");
1259}
1260
1261static bool convertIntrinsicValidType(StringRef Name,
1262 const FunctionType *FuncTy) {
1263 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1264 if (Name.starts_with("to.fp16")) {
1265 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1266 HalfTy) &&
1267 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1268 FuncTy->getReturnType());
1269 }
1270
1271 if (Name.starts_with("from.fp16")) {
1272 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1273 HalfTy) &&
1274 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1275 FuncTy->getReturnType());
1276 }
1277
1278 return false;
1279}
1280
1281static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1282 bool CanUpgradeDebugIntrinsicsToRecords) {
1283 assert(F && "Illegal to upgrade a non-existent Function.");
1284
1285 StringRef Name = F->getName();
1286
1287 // Quickly eliminate it, if it's not a candidate.
1288 if (!Name.consume_front("llvm.") || Name.empty())
1289 return false;
1290
1291 switch (Name[0]) {
1292 default: break;
1293 case 'a': {
1294 bool IsArm = Name.consume_front("arm.");
1295 if (IsArm || Name.consume_front("aarch64.")) {
1296 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1297 return true;
1298 break;
1299 }
1300
1301 if (Name.consume_front("amdgcn.")) {
1302 if (Name == "alignbit") {
1303 // Target specific intrinsic became redundant
1304 NewFn = Intrinsic::getOrInsertDeclaration(
1305 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1306 return true;
1307 }
1308
1309 if (Name.consume_front("atomic.")) {
1310 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1311 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1312 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1313 // and usub_sat so there's no new declaration.
1314 NewFn = nullptr;
1315 return true;
1316 }
1317 break; // No other 'amdgcn.atomic.*'
1318 }
1319
1320 switch (F->getIntrinsicID()) {
1321 default:
1322 break;
1323 // Legacy wmma iu intrinsics without the optional clamp operand.
1324 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1325 if (F->arg_size() == 7) {
1326 NewFn = nullptr;
1327 return true;
1328 }
1329 break;
1330 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1331 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1332 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1333 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1334 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1335 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1336 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1337 if (F->arg_size() == 8) {
1338 NewFn = nullptr;
1339 return true;
1340 }
1341 break;
1342 }
1343
1344 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1345 Name.consume_front("flat.atomic.")) {
1346 if (Name.starts_with("fadd") ||
1347 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1348 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1349 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1350 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1351 // declaration.
1352 NewFn = nullptr;
1353 return true;
1354 }
1355 }
1356
1357 if (Name.starts_with("ldexp.")) {
1358 // Target specific intrinsic became redundant
1359 NewFn = Intrinsic::getOrInsertDeclaration(
1360 F->getParent(), Intrinsic::ldexp,
1361 {F->getReturnType(), F->getArg(1)->getType()});
1362 return true;
1363 }
1364 break; // No other 'amdgcn.*'
1365 }
1366
1367 break;
1368 }
1369 case 'c': {
1370 if (F->arg_size() == 1) {
1371 if (Name.consume_front("convert.")) {
1372 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1373 NewFn = nullptr;
1374 return true;
1375 }
1376 }
1377
1378 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1379 .StartsWith("ctlz.", Intrinsic::ctlz)
1380 .StartsWith("cttz.", Intrinsic::cttz)
1381 .Default(Intrinsic::not_intrinsic);
1382 if (ID != Intrinsic::not_intrinsic) {
1383 rename(F);
1384 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1385 F->arg_begin()->getType());
1386 return true;
1387 }
1388 }
1389
1390 if (F->arg_size() == 2 && Name == "coro.end") {
1391 rename(F);
1392 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1393 Intrinsic::coro_end);
1394 return true;
1395 }
1396
1397 break;
1398 }
1399 case 'd':
1400 if (Name.consume_front("dbg.")) {
1401 // Mark debug intrinsics for upgrade to new debug format.
1402 if (CanUpgradeDebugIntrinsicsToRecords) {
1403 if (Name == "addr" || Name == "value" || Name == "assign" ||
1404 Name == "declare" || Name == "label") {
1405 // There's no function to replace these with.
1406 NewFn = nullptr;
1407 // But we do want these to get upgraded.
1408 return true;
1409 }
1410 }
1411 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1412 // converted to DbgVariableRecords later.
1413 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1414 rename(F);
1415 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1416 Intrinsic::dbg_value);
1417 return true;
1418 }
1419 break; // No other 'dbg.*'.
1420 }
1421 break;
1422 case 'e':
1423 if (Name.consume_front("experimental.vector.")) {
1424 Intrinsic::ID ID =
1425 StringSwitch<Intrinsic::ID>(Name)
1426 // Skip over extract.last.active, otherwise it will be 'upgraded'
1427 // to a regular vector extract which is a different operation.
1428 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1429 .StartsWith("extract.", Intrinsic::vector_extract)
1430 .StartsWith("insert.", Intrinsic::vector_insert)
1431 .StartsWith("reverse.", Intrinsic::vector_reverse)
1432 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1433 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1434 .StartsWith("partial.reduce.add",
1435 Intrinsic::vector_partial_reduce_add)
1436 .Default(Intrinsic::not_intrinsic);
1437 if (ID != Intrinsic::not_intrinsic) {
1438 const auto *FT = F->getFunctionType();
1439 SmallVector<Type *, 2> Tys;
1440 if (ID == Intrinsic::vector_extract ||
1441 ID == Intrinsic::vector_interleave2)
1442 // Extracting overloads the return type.
1443 Tys.push_back(FT->getReturnType());
1444 if (ID != Intrinsic::vector_interleave2)
1445 Tys.push_back(FT->getParamType(0));
1446 if (ID == Intrinsic::vector_insert ||
1447 ID == Intrinsic::vector_partial_reduce_add)
1448 // Inserting overloads the inserted type.
1449 Tys.push_back(FT->getParamType(1));
1450 rename(F);
1451 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1452 return true;
1453 }
1454
1455 if (Name.consume_front("reduce.")) {
1456 SmallVector<StringRef, 2> Groups;
1457 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1458 if (R.match(Name, &Groups))
1459 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1460 .Case("add", Intrinsic::vector_reduce_add)
1461 .Case("mul", Intrinsic::vector_reduce_mul)
1462 .Case("and", Intrinsic::vector_reduce_and)
1463 .Case("or", Intrinsic::vector_reduce_or)
1464 .Case("xor", Intrinsic::vector_reduce_xor)
1465 .Case("smax", Intrinsic::vector_reduce_smax)
1466 .Case("smin", Intrinsic::vector_reduce_smin)
1467 .Case("umax", Intrinsic::vector_reduce_umax)
1468 .Case("umin", Intrinsic::vector_reduce_umin)
1469 .Case("fmax", Intrinsic::vector_reduce_fmax)
1470 .Case("fmin", Intrinsic::vector_reduce_fmin)
1471 .Default(Intrinsic::not_intrinsic);
1472
1473 bool V2 = false;
1474 if (ID == Intrinsic::not_intrinsic) {
1475 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1476 Groups.clear();
1477 V2 = true;
1478 if (R2.match(Name, &Groups))
1479 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1480 .Case("fadd", Intrinsic::vector_reduce_fadd)
1481 .Case("fmul", Intrinsic::vector_reduce_fmul)
1482 .Default(Intrinsic::not_intrinsic);
1483 }
1484 if (ID != Intrinsic::not_intrinsic) {
1485 rename(F);
1486 auto Args = F->getFunctionType()->params();
1487 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1488 {Args[V2 ? 1 : 0]});
1489 return true;
1490 }
1491 break; // No other 'expermental.vector.reduce.*'.
1492 }
1493
1494 if (Name.consume_front("splice"))
1495 return true;
1496 break; // No other 'experimental.vector.*'.
1497 }
1498 if (Name.consume_front("experimental.stepvector.")) {
1499 Intrinsic::ID ID = Intrinsic::stepvector;
1500 rename(F);
1501 NewFn = Intrinsic::getOrInsertDeclaration(
1502 F->getParent(), ID, F->getFunctionType()->getReturnType());
1503 return true;
1504 }
1505 break; // No other 'e*'.
1506 case 'f':
1507 if (Name.starts_with("flt.rounds")) {
1508 rename(F);
1509 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1510 Intrinsic::get_rounding);
1511 return true;
1512 }
1513 break;
1514 case 'i':
1515 if (Name.starts_with("invariant.group.barrier")) {
1516 // Rename invariant.group.barrier to launder.invariant.group
1517 auto Args = F->getFunctionType()->params();
1518 Type* ObjectPtr[1] = {Args[0]};
1519 rename(F);
1520 NewFn = Intrinsic::getOrInsertDeclaration(
1521 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1522 return true;
1523 }
1524 break;
1525 case 'l':
1526 if ((Name.starts_with("lifetime.start") ||
1527 Name.starts_with("lifetime.end")) &&
1528 F->arg_size() == 2) {
1529 Intrinsic::ID IID = Name.starts_with("lifetime.start")
1530 ? Intrinsic::lifetime_start
1531 : Intrinsic::lifetime_end;
1532 rename(F);
1533 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1534 F->getArg(0)->getType());
1535 return true;
1536 }
1537 break;
1538 case 'm': {
1539 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1540 // alignment parameter to embedding the alignment as an attribute of
1541 // the pointer args.
1542 if (unsigned ID = StringSwitch<unsigned>(Name)
1543 .StartsWith("memcpy.", Intrinsic::memcpy)
1544 .StartsWith("memmove.", Intrinsic::memmove)
1545 .Default(0)) {
1546 if (F->arg_size() == 5) {
1547 rename(F);
1548 // Get the types of dest, src, and len
1549 ArrayRef<Type *> ParamTypes =
1550 F->getFunctionType()->params().slice(0, 3);
1551 NewFn =
1552 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1553 return true;
1554 }
1555 }
1556 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1557 rename(F);
1558 // Get the types of dest, and len
1559 const auto *FT = F->getFunctionType();
1560 Type *ParamTypes[2] = {
1561 FT->getParamType(0), // Dest
1562 FT->getParamType(2) // len
1563 };
1564 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1565 Intrinsic::memset, ParamTypes);
1566 return true;
1567 }
1568
1569 unsigned MaskedID =
1570 StringSwitch<unsigned>(Name)
1571 .StartsWith("masked.load", Intrinsic::masked_load)
1572 .StartsWith("masked.gather", Intrinsic::masked_gather)
1573 .StartsWith("masked.store", Intrinsic::masked_store)
1574 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1575 .Default(0);
1576 if (MaskedID && F->arg_size() == 4) {
1577 rename(F);
1578 if (MaskedID == Intrinsic::masked_load ||
1579 MaskedID == Intrinsic::masked_gather) {
1580 NewFn = Intrinsic::getOrInsertDeclaration(
1581 F->getParent(), MaskedID,
1582 {F->getReturnType(), F->getArg(0)->getType()});
1583 return true;
1584 }
1585 NewFn = Intrinsic::getOrInsertDeclaration(
1586 F->getParent(), MaskedID,
1587 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1588 return true;
1589 }
1590 break;
1591 }
1592 case 'n': {
1593 if (Name.consume_front("nvvm.")) {
1594 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1595 if (F->arg_size() == 1) {
1596 Intrinsic::ID IID =
1597 StringSwitch<Intrinsic::ID>(Name)
1598 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1599 .Case("clz.i", Intrinsic::ctlz)
1600 .Case("popc.i", Intrinsic::ctpop)
1601 .Default(Intrinsic::not_intrinsic);
1602 if (IID != Intrinsic::not_intrinsic) {
1603 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1604 {F->getReturnType()});
1605 return true;
1606 }
1607 } else if (F->arg_size() == 2) {
1608 Intrinsic::ID IID =
1609 StringSwitch<Intrinsic::ID>(Name)
1610 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1611 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1612 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1613 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1614 .Default(Intrinsic::not_intrinsic);
1615 if (IID != Intrinsic::not_intrinsic) {
1616 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1617 {F->getReturnType()});
1618 return true;
1619 }
1620 }
1621
1622 // Check for nvvm intrinsics that need a return type adjustment.
1623 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1624 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1625 if (IID != Intrinsic::not_intrinsic) {
1626 NewFn = nullptr;
1627 return true;
1628 }
1629 }
1630
1631 // Upgrade Distributed Shared Memory Intrinsics
1632 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1633 if (IID != Intrinsic::not_intrinsic) {
1634 rename(F);
1635 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1636 return true;
1637 }
1638
1639 // Upgrade TMA copy G2S Intrinsics
1640 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1641 if (IID != Intrinsic::not_intrinsic) {
1642 rename(F);
1643 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1644 return true;
1645 }
1646
1647 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1648 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1649 //
1650 // TODO: We could add lohi.i2d.
1651 bool Expand = false;
1652 if (Name.consume_front("abs."))
1653 // nvvm.abs.{i,ii}
1654 Expand =
1655 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1656 else if (Name.consume_front("fabs."))
1657 // nvvm.fabs.{f,ftz.f,d}
1658 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1659 else if (Name.consume_front("ex2.approx."))
1660 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1661 Expand =
1662 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1663 else if (Name.consume_front("atomic.load."))
1664 // nvvm.atomic.load.add.{f32,f64}.p
1665 // nvvm.atomic.load.{inc,dec}.32.p
1666 Expand = StringSwitch<bool>(Name)
1667 .StartsWith("add.f32.p", true)
1668 .StartsWith("add.f64.p", true)
1669 .StartsWith("inc.32.p", true)
1670 .StartsWith("dec.32.p", true)
1671 .Default(false);
1672 else if (Name.consume_front("bitcast."))
1673 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
1674 Expand =
1675 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
1676 else if (Name.consume_front("rotate."))
1677 // nvvm.rotate.{b32,b64,right.b64}
1678 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
1679 else if (Name.consume_front("ptr.gen.to."))
1680 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
1681 Expand = consumeNVVMPtrAddrSpace(Name);
1682 else if (Name.consume_front("ptr."))
1683 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
1684 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
1685 else if (Name.consume_front("ldg.global."))
1686 // nvvm.ldg.global.{i,p,f}
1687 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
1688 Name.starts_with("p."));
1689 else
1690 Expand = StringSwitch<bool>(Name)
1691 .Case("barrier0", true)
1692 .Case("barrier.n", true)
1693 .Case("barrier.sync.cnt", true)
1694 .Case("barrier.sync", true)
1695 .Case("barrier", true)
1696 .Case("bar.sync", true)
1697 .Case("barrier0.popc", true)
1698 .Case("barrier0.and", true)
1699 .Case("barrier0.or", true)
1700 .Case("clz.ll", true)
1701 .Case("popc.ll", true)
1702 .Case("h2f", true)
1703 .Case("swap.lo.hi.b64", true)
1704 .Case("tanh.approx.f32", true)
1705 .Default(false);
1706
1707 if (Expand) {
1708 NewFn = nullptr;
1709 return true;
1710 }
1711 break; // No other 'nvvm.*'.
1712 }
1713 break;
1714 }
1715 case 'o':
1716 if (Name.starts_with("objectsize.")) {
1717 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1718 if (F->arg_size() == 2 || F->arg_size() == 3) {
1719 rename(F);
1720 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1721 Intrinsic::objectsize, Tys);
1722 return true;
1723 }
1724 }
1725 break;
1726
1727 case 'p':
1728 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
1729 rename(F);
1730 NewFn = Intrinsic::getOrInsertDeclaration(
1731 F->getParent(), Intrinsic::ptr_annotation,
1732 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1733 return true;
1734 }
1735 break;
1736
1737 case 'r': {
1738 if (Name.consume_front("riscv.")) {
1739 Intrinsic::ID ID;
1740 ID = StringSwitch<Intrinsic::ID>(Name)
1741 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
1742 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
1743 .Case("aes32esi", Intrinsic::riscv_aes32esi)
1744 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
1745 .Default(Intrinsic::not_intrinsic);
1746 if (ID != Intrinsic::not_intrinsic) {
1747 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1748 rename(F);
1749 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1750 return true;
1751 }
1752 break; // No other applicable upgrades.
1753 }
1754
1755 ID = StringSwitch<Intrinsic::ID>(Name)
1756 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
1757 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
1758 .Default(Intrinsic::not_intrinsic);
1759 if (ID != Intrinsic::not_intrinsic) {
1760 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1761 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1762 rename(F);
1763 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1764 return true;
1765 }
1766 break; // No other applicable upgrades.
1767 }
1768
1769 ID = StringSwitch<Intrinsic::ID>(Name)
1770 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
1771 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
1772 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
1773 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
1774 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
1775 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
1776 .Default(Intrinsic::not_intrinsic);
1777 if (ID != Intrinsic::not_intrinsic) {
1778 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1779 rename(F);
1780 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1781 return true;
1782 }
1783 break; // No other applicable upgrades.
1784 }
1785
1786 // Replace llvm.riscv.clmul with llvm.clmul.
1787 if (Name == "clmul.i32" || Name == "clmul.i64") {
1788 NewFn = Intrinsic::getOrInsertDeclaration(
1789 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
1790 return true;
1791 }
1792
1793 break; // No other 'riscv.*' intrinsics
1794 }
1795 } break;
1796
1797 case 's':
1798 if (Name == "stackprotectorcheck") {
1799 NewFn = nullptr;
1800 return true;
1801 }
1802 break;
1803
1804 case 't':
1805 if (Name == "thread.pointer") {
1806 NewFn = Intrinsic::getOrInsertDeclaration(
1807 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
1808 return true;
1809 }
1810 break;
1811
1812 case 'v': {
1813 if (Name == "var.annotation" && F->arg_size() == 4) {
1814 rename(F);
1815 NewFn = Intrinsic::getOrInsertDeclaration(
1816 F->getParent(), Intrinsic::var_annotation,
1817 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1818 return true;
1819 }
1820 if (Name.consume_front("vector.splice")) {
1821 if (Name.starts_with(".left") || Name.starts_with(".right"))
1822 break;
1823 return true;
1824 }
1825 break;
1826 }
1827
1828 case 'w':
1829 if (Name.consume_front("wasm.")) {
1830 Intrinsic::ID ID =
1831 StringSwitch<Intrinsic::ID>(Name)
1832 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
1833 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
1834 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
1835 .Default(Intrinsic::not_intrinsic);
1836 if (ID != Intrinsic::not_intrinsic) {
1837 rename(F);
1838 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1839 F->getReturnType());
1840 return true;
1841 }
1842
1843 if (Name.consume_front("dot.i8x16.i7x16.")) {
1844 ID = StringSwitch<Intrinsic::ID>(Name)
1845 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1846 .Case("add.signed",
1847 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1848 .Default(Intrinsic::not_intrinsic);
1849 if (ID != Intrinsic::not_intrinsic) {
1850 rename(F);
1851 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1852 return true;
1853 }
1854 break; // No other 'wasm.dot.i8x16.i7x16.*'.
1855 }
1856 break; // No other 'wasm.*'.
1857 }
1858 break;
1859
1860 case 'x':
1861 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
1862 return true;
1863 }
1864
1865 auto *ST = dyn_cast<StructType>(F->getReturnType());
1866 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
1867 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
1868 // Replace return type with literal non-packed struct. Only do this for
1869 // intrinsics declared to return a struct, not for intrinsics with
1870 // overloaded return type, in which case the exact struct type will be
1871 // mangled into the name.
1872 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
1873 FunctionType *FT = F->getFunctionType();
1874 auto *NewST = StructType::get(ST->getContext(), ST->elements());
1875 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1876 std::string Name = F->getName().str();
1877 rename(F);
1878 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1879 Name, F->getParent());
1880
1881 // The new function may also need remangling.
1882 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
1883 NewFn = *Result;
1884 return true;
1885 }
1886 }
1887
1888 // Remangle our intrinsic since we upgrade the mangling
1889 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1890 if (Result != std::nullopt) {
1891 NewFn = *Result;
1892 return true;
1893 }
1894
1895 // This may not belong here. This function is effectively being overloaded
1896 // to both detect an intrinsic which needs upgrading, and to provide the
1897 // upgraded form of the intrinsic. We should perhaps have two separate
1898 // functions for this.
1899 return false;
1900}
1901
1902bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
1903 bool CanUpgradeDebugIntrinsicsToRecords) {
1904 NewFn = nullptr;
1905 bool Upgraded =
1906 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
1907
1908 // Upgrade intrinsic attributes. This does not change the function.
1909 if (NewFn)
1910 F = NewFn;
1911 if (Intrinsic::ID id = F->getIntrinsicID()) {
1912 // Only do this if the intrinsic signature is valid.
1913 SmallVector<Type *> OverloadTys;
1914 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
1915 F->setAttributes(
1916 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
1917 }
1918 return Upgraded;
1919}
1920
1921GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
1922 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
1923 GV->getName() == "llvm.global_dtors")) ||
1924 !GV->hasInitializer())
1925 return nullptr;
1926 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1927 if (!ATy)
1928 return nullptr;
1929 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1930 if (!STy || STy->getNumElements() != 2)
1931 return nullptr;
1932
1933 LLVMContext &C = GV->getContext();
1934 IRBuilder<> IRB(C);
1935 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1936 IRB.getPtrTy());
1937 Constant *Init = GV->getInitializer();
1938 unsigned N = Init->getNumOperands();
1939 std::vector<Constant *> NewCtors(N);
1940 for (unsigned i = 0; i != N; ++i) {
1941 auto Ctor = cast<Constant>(Init->getOperand(i));
1942 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
1943 Ctor->getAggregateElement(1),
1944 ConstantPointerNull::get(IRB.getPtrTy()));
1945 }
1946 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1947
1948 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1949 NewInit, GV->getName());
1950}
1951
1952// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1953// to byte shuffles.
1954static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1955 unsigned Shift) {
1956 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1957 unsigned NumElts = ResultTy->getNumElements() * 8;
1958
1959 // Bitcast from a 64-bit element type to a byte element type.
1960 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1961 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1962
1963 // We'll be shuffling in zeroes.
1964 Value *Res = Constant::getNullValue(VecTy);
1965
1966 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1967 // we'll just return the zero vector.
1968 if (Shift < 16) {
1969 int Idxs[64];
1970 // 256/512-bit version is split into 2/4 16-byte lanes.
1971 for (unsigned l = 0; l != NumElts; l += 16)
1972 for (unsigned i = 0; i != 16; ++i) {
1973 unsigned Idx = NumElts + i - Shift;
1974 if (Idx < NumElts)
1975 Idx -= NumElts - 16; // end of lane, switch operand.
1976 Idxs[l + i] = Idx + l;
1977 }
1978
1979 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
1980 }
1981
1982 // Bitcast back to a 64-bit element type.
1983 return Builder.CreateBitCast(Res, ResultTy, "cast");
1984}
1985
1986// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1987// to byte shuffles.
1988static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1989 unsigned Shift) {
1990 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1991 unsigned NumElts = ResultTy->getNumElements() * 8;
1992
1993 // Bitcast from a 64-bit element type to a byte element type.
1994 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1995 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1996
1997 // We'll be shuffling in zeroes.
1998 Value *Res = Constant::getNullValue(VecTy);
1999
2000 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2001 // we'll just return the zero vector.
2002 if (Shift < 16) {
2003 int Idxs[64];
2004 // 256/512-bit version is split into 2/4 16-byte lanes.
2005 for (unsigned l = 0; l != NumElts; l += 16)
2006 for (unsigned i = 0; i != 16; ++i) {
2007 unsigned Idx = i + Shift;
2008 if (Idx >= 16)
2009 Idx += NumElts - 16; // end of lane, switch operand.
2010 Idxs[l + i] = Idx + l;
2011 }
2012
2013 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2014 }
2015
2016 // Bitcast back to a 64-bit element type.
2017 return Builder.CreateBitCast(Res, ResultTy, "cast");
2018}
2019
2020static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2021 unsigned NumElts) {
2022 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2023 llvm::VectorType *MaskTy = FixedVectorType::get(
2024 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2025 Mask = Builder.CreateBitCast(Mask, MaskTy);
2026
2027 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2028 // i8 and we need to extract down to the right number of elements.
2029 if (NumElts <= 4) {
2030 int Indices[4];
2031 for (unsigned i = 0; i != NumElts; ++i)
2032 Indices[i] = i;
2033 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2034 "extract");
2035 }
2036
2037 return Mask;
2038}
2039
2040static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2041 Value *Op1) {
2042 // If the mask is all ones just emit the first operation.
2043 if (const auto *C = dyn_cast<Constant>(Mask))
2044 if (C->isAllOnesValue())
2045 return Op0;
2046
2047 Mask = getX86MaskVec(Builder, Mask,
2048 cast<FixedVectorType>(Op0->getType())->getNumElements());
2049 return Builder.CreateSelect(Mask, Op0, Op1);
2050}
2051
2052static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2053 Value *Op1) {
2054 // If the mask is all ones just emit the first operation.
2055 if (const auto *C = dyn_cast<Constant>(Mask))
2056 if (C->isAllOnesValue())
2057 return Op0;
2058
2059 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2060 Mask->getType()->getIntegerBitWidth());
2061 Mask = Builder.CreateBitCast(Mask, MaskTy);
2062 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2063 return Builder.CreateSelect(Mask, Op0, Op1);
2064}
2065
2066// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2067// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2068// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2069static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2070 Value *Op1, Value *Shift,
2071 Value *Passthru, Value *Mask,
2072 bool IsVALIGN) {
2073 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2074
2075 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2076 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2077 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2078 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2079
2080 // Mask the immediate for VALIGN.
2081 if (IsVALIGN)
2082 ShiftVal &= (NumElts - 1);
2083
2084 // If palignr is shifting the pair of vectors more than the size of two
2085 // lanes, emit zero.
2086 if (ShiftVal >= 32)
2087 return llvm::Constant::getNullValue(Op0->getType());
2088
2089 // If palignr is shifting the pair of input vectors more than one lane,
2090 // but less than two lanes, convert to shifting in zeroes.
2091 if (ShiftVal > 16) {
2092 ShiftVal -= 16;
2093 Op1 = Op0;
2094 Op0 = llvm::Constant::getNullValue(Op0->getType());
2095 }
2096
2097 int Indices[64];
2098 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2099 for (unsigned l = 0; l < NumElts; l += 16) {
2100 for (unsigned i = 0; i != 16; ++i) {
2101 unsigned Idx = ShiftVal + i;
2102 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2103 Idx += NumElts - 16; // End of lane, switch operand.
2104 Indices[l + i] = Idx + l;
2105 }
2106 }
2107
2108 Value *Align = Builder.CreateShuffleVector(
2109 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2110
2111 return emitX86Select(Builder, Mask, Align, Passthru);
2112}
2113
2114static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2115 bool ZeroMask, bool IndexForm) {
2116 Type *Ty = CI.getType();
2117 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2118 unsigned EltWidth = Ty->getScalarSizeInBits();
2119 bool IsFloat = Ty->isFPOrFPVectorTy();
2120 Intrinsic::ID IID;
2121 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2122 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2123 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2124 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2125 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2126 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2127 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2128 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2129 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2130 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2131 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2132 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2133 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2134 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2135 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2136 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2137 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2138 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2139 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2140 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2141 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2142 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2143 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2144 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2145 else if (VecWidth == 128 && EltWidth == 16)
2146 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2147 else if (VecWidth == 256 && EltWidth == 16)
2148 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2149 else if (VecWidth == 512 && EltWidth == 16)
2150 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2151 else if (VecWidth == 128 && EltWidth == 8)
2152 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2153 else if (VecWidth == 256 && EltWidth == 8)
2154 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2155 else if (VecWidth == 512 && EltWidth == 8)
2156 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2157 else
2158 llvm_unreachable("Unexpected intrinsic");
2159
2160 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2161 CI.getArgOperand(2) };
2162
2163 // If this isn't index form we need to swap operand 0 and 1.
2164 if (!IndexForm)
2165 std::swap(Args[0], Args[1]);
2166
2167 Value *V = Builder.CreateIntrinsic(IID, Args);
2168 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2169 : Builder.CreateBitCast(CI.getArgOperand(1),
2170 Ty);
2171 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2172}
2173
2174static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2175 Intrinsic::ID IID) {
2176 Type *Ty = CI.getType();
2177 Value *Op0 = CI.getOperand(0);
2178 Value *Op1 = CI.getOperand(1);
2179 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2180
2181 if (CI.arg_size() == 4) { // For masked intrinsics.
2182 Value *VecSrc = CI.getOperand(2);
2183 Value *Mask = CI.getOperand(3);
2184 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2185 }
2186 return Res;
2187}
2188
2189static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2190 bool IsRotateRight) {
2191 Type *Ty = CI.getType();
2192 Value *Src = CI.getArgOperand(0);
2193 Value *Amt = CI.getArgOperand(1);
2194
2195 // Amount may be scalar immediate, in which case create a splat vector.
2196 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2197 // we only care about the lowest log2 bits anyway.
2198 if (Amt->getType() != Ty) {
2199 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2200 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2201 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2202 }
2203
2204 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2205 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2206
2207 if (CI.arg_size() == 4) { // For masked intrinsics.
2208 Value *VecSrc = CI.getOperand(2);
2209 Value *Mask = CI.getOperand(3);
2210 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2211 }
2212 return Res;
2213}
2214
2215static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2216 bool IsSigned) {
2217 Type *Ty = CI.getType();
2218 Value *LHS = CI.getArgOperand(0);
2219 Value *RHS = CI.getArgOperand(1);
2220
2221 CmpInst::Predicate Pred;
2222 switch (Imm) {
2223 case 0x0:
2224 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2225 break;
2226 case 0x1:
2227 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2228 break;
2229 case 0x2:
2230 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2231 break;
2232 case 0x3:
2233 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2234 break;
2235 case 0x4:
2236 Pred = ICmpInst::ICMP_EQ;
2237 break;
2238 case 0x5:
2239 Pred = ICmpInst::ICMP_NE;
2240 break;
2241 case 0x6:
2242 return Constant::getNullValue(Ty); // FALSE
2243 case 0x7:
2244 return Constant::getAllOnesValue(Ty); // TRUE
2245 default:
2246 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2247 }
2248
2249 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2250 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2251 return Ext;
2252}
2253
2254static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2255 bool IsShiftRight, bool ZeroMask) {
2256 Type *Ty = CI.getType();
2257 Value *Op0 = CI.getArgOperand(0);
2258 Value *Op1 = CI.getArgOperand(1);
2259 Value *Amt = CI.getArgOperand(2);
2260
2261 if (IsShiftRight)
2262 std::swap(Op0, Op1);
2263
2264 // Amount may be scalar immediate, in which case create a splat vector.
2265 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2266 // we only care about the lowest log2 bits anyway.
2267 if (Amt->getType() != Ty) {
2268 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2269 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2270 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2271 }
2272
2273 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2274 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2275
2276 unsigned NumArgs = CI.arg_size();
2277 if (NumArgs >= 4) { // For masked intrinsics.
2278 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2279 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2280 CI.getArgOperand(0);
2281 Value *Mask = CI.getOperand(NumArgs - 1);
2282 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2283 }
2284 return Res;
2285}
2286
2287static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2288 Value *Mask, bool Aligned) {
2289 const Align Alignment =
2290 Aligned
2291 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2292 : Align(1);
2293
2294 // If the mask is all ones just emit a regular store.
2295 if (const auto *C = dyn_cast<Constant>(Mask))
2296 if (C->isAllOnesValue())
2297 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2298
2299 // Convert the mask from an integer type to a vector of i1.
2300 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2301 Mask = getX86MaskVec(Builder, Mask, NumElts);
2302 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2303}
2304
2305static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2306 Value *Passthru, Value *Mask, bool Aligned) {
2307 Type *ValTy = Passthru->getType();
2308 const Align Alignment =
2309 Aligned
2310 ? Align(
2311 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2312 8)
2313 : Align(1);
2314
2315 // If the mask is all ones just emit a regular store.
2316 if (const auto *C = dyn_cast<Constant>(Mask))
2317 if (C->isAllOnesValue())
2318 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2319
2320 // Convert the mask from an integer type to a vector of i1.
2321 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2322 Mask = getX86MaskVec(Builder, Mask, NumElts);
2323 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2324}
2325
2326static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2327 Type *Ty = CI.getType();
2328 Value *Op0 = CI.getArgOperand(0);
2329 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2330 {Op0, Builder.getInt1(false)});
2331 if (CI.arg_size() == 3)
2332 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2333 return Res;
2334}
2335
2336static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2337 Type *Ty = CI.getType();
2338
2339 // Arguments have a vXi32 type so cast to vXi64.
2340 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2341 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2342
2343 if (IsSigned) {
2344 // Shift left then arithmetic shift right.
2345 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2346 LHS = Builder.CreateShl(LHS, ShiftAmt);
2347 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2348 RHS = Builder.CreateShl(RHS, ShiftAmt);
2349 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2350 } else {
2351 // Clear the upper bits.
2352 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2353 LHS = Builder.CreateAnd(LHS, Mask);
2354 RHS = Builder.CreateAnd(RHS, Mask);
2355 }
2356
2357 Value *Res = Builder.CreateMul(LHS, RHS);
2358
2359 if (CI.arg_size() == 4)
2360 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2361
2362 return Res;
2363}
2364
2365// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2366static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2367 Value *Mask) {
2368 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2369 if (Mask) {
2370 const auto *C = dyn_cast<Constant>(Mask);
2371 if (!C || !C->isAllOnesValue())
2372 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2373 }
2374
2375 if (NumElts < 8) {
2376 int Indices[8];
2377 for (unsigned i = 0; i != NumElts; ++i)
2378 Indices[i] = i;
2379 for (unsigned i = NumElts; i != 8; ++i)
2380 Indices[i] = NumElts + i % NumElts;
2381 Vec = Builder.CreateShuffleVector(Vec,
2382 Constant::getNullValue(Vec->getType()),
2383 Indices);
2384 }
2385 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2386}
2387
2388static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2389 unsigned CC, bool Signed) {
2390 Value *Op0 = CI.getArgOperand(0);
2391 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2392
2393 Value *Cmp;
2394 if (CC == 3) {
2395 Cmp = Constant::getNullValue(
2396 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2397 } else if (CC == 7) {
2398 Cmp = Constant::getAllOnesValue(
2399 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2400 } else {
2401 ICmpInst::Predicate Pred;
2402 switch (CC) {
2403 default: llvm_unreachable("Unknown condition code");
2404 case 0: Pred = ICmpInst::ICMP_EQ; break;
2405 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2406 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2407 case 4: Pred = ICmpInst::ICMP_NE; break;
2408 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2409 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2410 }
2411 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2412 }
2413
2414 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2415
2416 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2417}
2418
2419// Replace a masked intrinsic with an older unmasked intrinsic.
2420static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2421 Intrinsic::ID IID) {
2422 Value *Rep =
2423 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2424 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2425}
2426
2427static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2428 Value* A = CI.getArgOperand(0);
2429 Value* B = CI.getArgOperand(1);
2430 Value* Src = CI.getArgOperand(2);
2431 Value* Mask = CI.getArgOperand(3);
2432
2433 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2434 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2435 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2436 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2437 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2438 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2439}
2440
2441static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2442 Value* Op = CI.getArgOperand(0);
2443 Type* ReturnOp = CI.getType();
2444 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2445 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2446 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2447}
2448
2449// Replace intrinsic with unmasked version and a select.
2450static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2451 CallBase &CI, Value *&Rep) {
2452 Name = Name.substr(12); // Remove avx512.mask.
2453
2454 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2455 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2456 Intrinsic::ID IID;
2457 if (Name.starts_with("max.p")) {
2458 if (VecWidth == 128 && EltWidth == 32)
2459 IID = Intrinsic::x86_sse_max_ps;
2460 else if (VecWidth == 128 && EltWidth == 64)
2461 IID = Intrinsic::x86_sse2_max_pd;
2462 else if (VecWidth == 256 && EltWidth == 32)
2463 IID = Intrinsic::x86_avx_max_ps_256;
2464 else if (VecWidth == 256 && EltWidth == 64)
2465 IID = Intrinsic::x86_avx_max_pd_256;
2466 else
2467 llvm_unreachable("Unexpected intrinsic");
2468 } else if (Name.starts_with("min.p")) {
2469 if (VecWidth == 128 && EltWidth == 32)
2470 IID = Intrinsic::x86_sse_min_ps;
2471 else if (VecWidth == 128 && EltWidth == 64)
2472 IID = Intrinsic::x86_sse2_min_pd;
2473 else if (VecWidth == 256 && EltWidth == 32)
2474 IID = Intrinsic::x86_avx_min_ps_256;
2475 else if (VecWidth == 256 && EltWidth == 64)
2476 IID = Intrinsic::x86_avx_min_pd_256;
2477 else
2478 llvm_unreachable("Unexpected intrinsic");
2479 } else if (Name.starts_with("pshuf.b.")) {
2480 if (VecWidth == 128)
2481 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2482 else if (VecWidth == 256)
2483 IID = Intrinsic::x86_avx2_pshuf_b;
2484 else if (VecWidth == 512)
2485 IID = Intrinsic::x86_avx512_pshuf_b_512;
2486 else
2487 llvm_unreachable("Unexpected intrinsic");
2488 } else if (Name.starts_with("pmul.hr.sw.")) {
2489 if (VecWidth == 128)
2490 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2491 else if (VecWidth == 256)
2492 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2493 else if (VecWidth == 512)
2494 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2495 else
2496 llvm_unreachable("Unexpected intrinsic");
2497 } else if (Name.starts_with("pmulh.w.")) {
2498 if (VecWidth == 128)
2499 IID = Intrinsic::x86_sse2_pmulh_w;
2500 else if (VecWidth == 256)
2501 IID = Intrinsic::x86_avx2_pmulh_w;
2502 else if (VecWidth == 512)
2503 IID = Intrinsic::x86_avx512_pmulh_w_512;
2504 else
2505 llvm_unreachable("Unexpected intrinsic");
2506 } else if (Name.starts_with("pmulhu.w.")) {
2507 if (VecWidth == 128)
2508 IID = Intrinsic::x86_sse2_pmulhu_w;
2509 else if (VecWidth == 256)
2510 IID = Intrinsic::x86_avx2_pmulhu_w;
2511 else if (VecWidth == 512)
2512 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2513 else
2514 llvm_unreachable("Unexpected intrinsic");
2515 } else if (Name.starts_with("pmaddw.d.")) {
2516 if (VecWidth == 128)
2517 IID = Intrinsic::x86_sse2_pmadd_wd;
2518 else if (VecWidth == 256)
2519 IID = Intrinsic::x86_avx2_pmadd_wd;
2520 else if (VecWidth == 512)
2521 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2522 else
2523 llvm_unreachable("Unexpected intrinsic");
2524 } else if (Name.starts_with("pmaddubs.w.")) {
2525 if (VecWidth == 128)
2526 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2527 else if (VecWidth == 256)
2528 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2529 else if (VecWidth == 512)
2530 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2531 else
2532 llvm_unreachable("Unexpected intrinsic");
2533 } else if (Name.starts_with("packsswb.")) {
2534 if (VecWidth == 128)
2535 IID = Intrinsic::x86_sse2_packsswb_128;
2536 else if (VecWidth == 256)
2537 IID = Intrinsic::x86_avx2_packsswb;
2538 else if (VecWidth == 512)
2539 IID = Intrinsic::x86_avx512_packsswb_512;
2540 else
2541 llvm_unreachable("Unexpected intrinsic");
2542 } else if (Name.starts_with("packssdw.")) {
2543 if (VecWidth == 128)
2544 IID = Intrinsic::x86_sse2_packssdw_128;
2545 else if (VecWidth == 256)
2546 IID = Intrinsic::x86_avx2_packssdw;
2547 else if (VecWidth == 512)
2548 IID = Intrinsic::x86_avx512_packssdw_512;
2549 else
2550 llvm_unreachable("Unexpected intrinsic");
2551 } else if (Name.starts_with("packuswb.")) {
2552 if (VecWidth == 128)
2553 IID = Intrinsic::x86_sse2_packuswb_128;
2554 else if (VecWidth == 256)
2555 IID = Intrinsic::x86_avx2_packuswb;
2556 else if (VecWidth == 512)
2557 IID = Intrinsic::x86_avx512_packuswb_512;
2558 else
2559 llvm_unreachable("Unexpected intrinsic");
2560 } else if (Name.starts_with("packusdw.")) {
2561 if (VecWidth == 128)
2562 IID = Intrinsic::x86_sse41_packusdw;
2563 else if (VecWidth == 256)
2564 IID = Intrinsic::x86_avx2_packusdw;
2565 else if (VecWidth == 512)
2566 IID = Intrinsic::x86_avx512_packusdw_512;
2567 else
2568 llvm_unreachable("Unexpected intrinsic");
2569 } else if (Name.starts_with("vpermilvar.")) {
2570 if (VecWidth == 128 && EltWidth == 32)
2571 IID = Intrinsic::x86_avx_vpermilvar_ps;
2572 else if (VecWidth == 128 && EltWidth == 64)
2573 IID = Intrinsic::x86_avx_vpermilvar_pd;
2574 else if (VecWidth == 256 && EltWidth == 32)
2575 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2576 else if (VecWidth == 256 && EltWidth == 64)
2577 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2578 else if (VecWidth == 512 && EltWidth == 32)
2579 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2580 else if (VecWidth == 512 && EltWidth == 64)
2581 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2582 else
2583 llvm_unreachable("Unexpected intrinsic");
2584 } else if (Name == "cvtpd2dq.256") {
2585 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2586 } else if (Name == "cvtpd2ps.256") {
2587 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2588 } else if (Name == "cvttpd2dq.256") {
2589 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2590 } else if (Name == "cvttps2dq.128") {
2591 IID = Intrinsic::x86_sse2_cvttps2dq;
2592 } else if (Name == "cvttps2dq.256") {
2593 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2594 } else if (Name.starts_with("permvar.")) {
2595 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2596 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2597 IID = Intrinsic::x86_avx2_permps;
2598 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2599 IID = Intrinsic::x86_avx2_permd;
2600 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2601 IID = Intrinsic::x86_avx512_permvar_df_256;
2602 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2603 IID = Intrinsic::x86_avx512_permvar_di_256;
2604 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2605 IID = Intrinsic::x86_avx512_permvar_sf_512;
2606 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2607 IID = Intrinsic::x86_avx512_permvar_si_512;
2608 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2609 IID = Intrinsic::x86_avx512_permvar_df_512;
2610 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2611 IID = Intrinsic::x86_avx512_permvar_di_512;
2612 else if (VecWidth == 128 && EltWidth == 16)
2613 IID = Intrinsic::x86_avx512_permvar_hi_128;
2614 else if (VecWidth == 256 && EltWidth == 16)
2615 IID = Intrinsic::x86_avx512_permvar_hi_256;
2616 else if (VecWidth == 512 && EltWidth == 16)
2617 IID = Intrinsic::x86_avx512_permvar_hi_512;
2618 else if (VecWidth == 128 && EltWidth == 8)
2619 IID = Intrinsic::x86_avx512_permvar_qi_128;
2620 else if (VecWidth == 256 && EltWidth == 8)
2621 IID = Intrinsic::x86_avx512_permvar_qi_256;
2622 else if (VecWidth == 512 && EltWidth == 8)
2623 IID = Intrinsic::x86_avx512_permvar_qi_512;
2624 else
2625 llvm_unreachable("Unexpected intrinsic");
2626 } else if (Name.starts_with("dbpsadbw.")) {
2627 if (VecWidth == 128)
2628 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2629 else if (VecWidth == 256)
2630 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2631 else if (VecWidth == 512)
2632 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2633 else
2634 llvm_unreachable("Unexpected intrinsic");
2635 } else if (Name.starts_with("pmultishift.qb.")) {
2636 if (VecWidth == 128)
2637 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2638 else if (VecWidth == 256)
2639 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2640 else if (VecWidth == 512)
2641 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2642 else
2643 llvm_unreachable("Unexpected intrinsic");
2644 } else if (Name.starts_with("conflict.")) {
2645 if (Name[9] == 'd' && VecWidth == 128)
2646 IID = Intrinsic::x86_avx512_conflict_d_128;
2647 else if (Name[9] == 'd' && VecWidth == 256)
2648 IID = Intrinsic::x86_avx512_conflict_d_256;
2649 else if (Name[9] == 'd' && VecWidth == 512)
2650 IID = Intrinsic::x86_avx512_conflict_d_512;
2651 else if (Name[9] == 'q' && VecWidth == 128)
2652 IID = Intrinsic::x86_avx512_conflict_q_128;
2653 else if (Name[9] == 'q' && VecWidth == 256)
2654 IID = Intrinsic::x86_avx512_conflict_q_256;
2655 else if (Name[9] == 'q' && VecWidth == 512)
2656 IID = Intrinsic::x86_avx512_conflict_q_512;
2657 else
2658 llvm_unreachable("Unexpected intrinsic");
2659 } else if (Name.starts_with("pavg.")) {
2660 if (Name[5] == 'b' && VecWidth == 128)
2661 IID = Intrinsic::x86_sse2_pavg_b;
2662 else if (Name[5] == 'b' && VecWidth == 256)
2663 IID = Intrinsic::x86_avx2_pavg_b;
2664 else if (Name[5] == 'b' && VecWidth == 512)
2665 IID = Intrinsic::x86_avx512_pavg_b_512;
2666 else if (Name[5] == 'w' && VecWidth == 128)
2667 IID = Intrinsic::x86_sse2_pavg_w;
2668 else if (Name[5] == 'w' && VecWidth == 256)
2669 IID = Intrinsic::x86_avx2_pavg_w;
2670 else if (Name[5] == 'w' && VecWidth == 512)
2671 IID = Intrinsic::x86_avx512_pavg_w_512;
2672 else
2673 llvm_unreachable("Unexpected intrinsic");
2674 } else
2675 return false;
2676
2677 SmallVector<Value *, 4> Args(CI.args());
2678 Args.pop_back();
2679 Args.pop_back();
2680 Rep = Builder.CreateIntrinsic(IID, Args);
2681 unsigned NumArgs = CI.arg_size();
2682 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2683 CI.getArgOperand(NumArgs - 2));
2684 return true;
2685}
2686
2687/// Upgrade comment in call to inline asm that represents an objc retain release
2688/// marker.
2689void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2690 size_t Pos;
2691 if (AsmStr->find("mov\tfp") == 0 &&
2692 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
2693 (Pos = AsmStr->find("# marker")) != std::string::npos) {
2694 AsmStr->replace(Pos, 1, ";");
2695 }
2696}
2697
2698static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
2699 Function *F, IRBuilder<> &Builder) {
2700 Value *Rep = nullptr;
2701
2702 if (Name == "abs.i" || Name == "abs.ll") {
2703 Value *Arg = CI->getArgOperand(0);
2704 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
2705 {Arg, Builder.getTrue()},
2706 /*FMFSource=*/nullptr, "abs");
2707 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
2708 Type *Ty = (Name == "abs.bf16")
2709 ? Builder.getBFloatTy()
2710 : FixedVectorType::get(Builder.getBFloatTy(), 2);
2711 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
2712 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
2713 Rep = Builder.CreateBitCast(Abs, CI->getType());
2714 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
2715 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
2716 : Intrinsic::nvvm_fabs;
2717 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2718 } else if (Name.consume_front("ex2.approx.")) {
2719 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2720 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
2721 : Intrinsic::nvvm_ex2_approx;
2722 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2723 } else if (Name.starts_with("atomic.load.add.f32.p") ||
2724 Name.starts_with("atomic.load.add.f64.p")) {
2725 Value *Ptr = CI->getArgOperand(0);
2726 Value *Val = CI->getArgOperand(1);
2727 Rep = Builder.CreateAtomicRMW(
2728 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2729 CI->getContext().getOrInsertSyncScopeID("device"));
2730 // The default scope for atomic.load.* intrinsics is device
2731 // (= gpu scope in ptx), but the default LLVM atomic scope is
2732 // "system"
2733 } else if (Name.starts_with("atomic.load.inc.32.p") ||
2734 Name.starts_with("atomic.load.dec.32.p")) {
2735 Value *Ptr = CI->getArgOperand(0);
2736 Value *Val = CI->getArgOperand(1);
2737 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
2738 : AtomicRMWInst::UDecWrap;
2739 Rep = Builder.CreateAtomicRMW(
2740 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2741 CI->getContext().getOrInsertSyncScopeID("device"));
2742 // See comment above.
2743 } else if (Name == "clz.ll") {
2744 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
2745 Value *Arg = CI->getArgOperand(0);
2746 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
2747 {Arg, Builder.getFalse()},
2748 /*FMFSource=*/nullptr, "ctlz");
2749 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
2750 } else if (Name == "popc.ll") {
2751 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
2752 // i64.
2753 Value *Arg = CI->getArgOperand(0);
2754 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
2755 Arg, /*FMFSource=*/nullptr, "ctpop");
2756 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
2757 } else if (Name == "h2f") {
2758 Value *Cast =
2759 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
2760 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
2761 } else if (Name.consume_front("bitcast.") &&
2762 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
2763 Name == "d2ll")) {
2764 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
2765 } else if (Name == "rotate.b32") {
2766 Value *Arg = CI->getOperand(0);
2767 Value *ShiftAmt = CI->getOperand(1);
2768 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
2769 {Arg, Arg, ShiftAmt});
2770 } else if (Name == "rotate.b64") {
2771 Type *Int64Ty = Builder.getInt64Ty();
2772 Value *Arg = CI->getOperand(0);
2773 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2774 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2775 {Arg, Arg, ZExtShiftAmt});
2776 } else if (Name == "rotate.right.b64") {
2777 Type *Int64Ty = Builder.getInt64Ty();
2778 Value *Arg = CI->getOperand(0);
2779 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2780 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
2781 {Arg, Arg, ZExtShiftAmt});
2782 } else if (Name == "swap.lo.hi.b64") {
2783 Type *Int64Ty = Builder.getInt64Ty();
2784 Value *Arg = CI->getOperand(0);
2785 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2786 {Arg, Arg, Builder.getInt64(32)});
2787 } else if ((Name.consume_front("ptr.gen.to.") &&
2788 consumeNVVMPtrAddrSpace(Name)) ||
2789 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
2790 Name.starts_with(".to.gen"))) {
2791 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
2792 } else if (Name.consume_front("ldg.global")) {
2793 Value *Ptr = CI->getArgOperand(0);
2794 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
2795 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
2796 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
2797 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
2798 MDNode *MD = MDNode::get(Builder.getContext(), {});
2799 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
2800 return LD;
2801 } else if (Name == "tanh.approx.f32") {
2802 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
2803 FastMathFlags FMF;
2804 FMF.setApproxFunc();
2805 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
2806 FMF);
2807 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
2808 Value *Arg =
2809 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
2810 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
2811 {}, {Arg});
2812 } else if (Name == "barrier") {
2813 Rep = Builder.CreateIntrinsic(
2814 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
2815 {CI->getArgOperand(0), CI->getArgOperand(1)});
2816 } else if (Name == "barrier.sync") {
2817 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
2818 {CI->getArgOperand(0)});
2819 } else if (Name == "barrier.sync.cnt") {
2820 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
2821 {CI->getArgOperand(0), CI->getArgOperand(1)});
2822 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
2823 Name == "barrier0.or") {
2824 Value *C = CI->getArgOperand(0);
2825 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
2826
2827 Intrinsic::ID IID =
2828 StringSwitch<Intrinsic::ID>(Name)
2829 .Case("barrier0.popc",
2830 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
2831 .Case("barrier0.and",
2832 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
2833 .Case("barrier0.or",
2834 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
2835 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
2836 Rep = Builder.CreateZExt(Bar, CI->getType());
2837 } else {
2838 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2839 if (IID != Intrinsic::not_intrinsic &&
2840 !F->getReturnType()->getScalarType()->isBFloatTy()) {
2841 rename(F);
2842 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2843 SmallVector<Value *, 2> Args;
2844 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
2845 Value *Arg = CI->getArgOperand(I);
2846 Type *OldType = Arg->getType();
2847 Type *NewType = NewFn->getArg(I)->getType();
2848 Args.push_back(
2849 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
2850 ? Builder.CreateBitCast(Arg, NewType)
2851 : Arg);
2852 }
2853 Rep = Builder.CreateCall(NewFn, Args);
2854 if (F->getReturnType()->isIntegerTy())
2855 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
2856 }
2857 }
2858
2859 return Rep;
2860}
2861
2862static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2863 IRBuilder<> &Builder) {
2864 LLVMContext &C = F->getContext();
2865 Value *Rep = nullptr;
2866
2867 if (Name.starts_with("sse4a.movnt.")) {
2868 SmallVector<Metadata *, 1> Elts;
2869 Elts.push_back(
2870 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2871 MDNode *Node = MDNode::get(C, Elts);
2872
2873 Value *Arg0 = CI->getArgOperand(0);
2874 Value *Arg1 = CI->getArgOperand(1);
2875
2876 // Nontemporal (unaligned) store of the 0'th element of the float/double
2877 // vector.
2878 Value *Extract =
2879 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
2880
2881 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
2882 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2883 } else if (Name.starts_with("avx.movnt.") ||
2884 Name.starts_with("avx512.storent.")) {
2885 SmallVector<Metadata *, 1> Elts;
2886 Elts.push_back(
2887 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2888 MDNode *Node = MDNode::get(C, Elts);
2889
2890 Value *Arg0 = CI->getArgOperand(0);
2891 Value *Arg1 = CI->getArgOperand(1);
2892
2893 StoreInst *SI = Builder.CreateAlignedStore(
2894 Arg1, Arg0,
2895 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
2896 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2897 } else if (Name == "sse2.storel.dq") {
2898 Value *Arg0 = CI->getArgOperand(0);
2899 Value *Arg1 = CI->getArgOperand(1);
2900
2901 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
2902 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
2903 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
2904 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
2905 } else if (Name.starts_with("sse.storeu.") ||
2906 Name.starts_with("sse2.storeu.") ||
2907 Name.starts_with("avx.storeu.")) {
2908 Value *Arg0 = CI->getArgOperand(0);
2909 Value *Arg1 = CI->getArgOperand(1);
2910 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
2911 } else if (Name == "avx512.mask.store.ss") {
2912 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
2913 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2914 Mask, false);
2915 } else if (Name.starts_with("avx512.mask.store")) {
2916 // "avx512.mask.storeu." or "avx512.mask.store."
2917 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
2918 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2919 CI->getArgOperand(2), Aligned);
2920 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
2921 // Upgrade packed integer vector compare intrinsics to compare instructions.
2922 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
2923 bool CmpEq = Name[9] == 'e';
2924 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2925 CI->getArgOperand(0), CI->getArgOperand(1));
2926 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
2927 } else if (Name.starts_with("avx512.broadcastm")) {
2928 Type *ExtTy = Type::getInt32Ty(C);
2929 if (CI->getOperand(0)->getType()->isIntegerTy(8))
2930 ExtTy = Type::getInt64Ty(C);
2931 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
2932 ExtTy->getPrimitiveSizeInBits();
2933 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
2934 Rep = Builder.CreateVectorSplat(NumElts, Rep);
2935 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
2936 Value *Vec = CI->getArgOperand(0);
2937 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
2938 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
2939 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
2940 } else if (Name.starts_with("avx.sqrt.p") ||
2941 Name.starts_with("sse2.sqrt.p") ||
2942 Name.starts_with("sse.sqrt.p")) {
2943 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
2944 {CI->getArgOperand(0)});
2945 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
2946 if (CI->arg_size() == 4 &&
2947 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2948 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2949 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
2950 : Intrinsic::x86_avx512_sqrt_pd_512;
2951
2952 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
2953 Rep = Builder.CreateIntrinsic(IID, Args);
2954 } else {
2955 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
2956 {CI->getArgOperand(0)});
2957 }
2958 Rep =
2959 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2960 } else if (Name.starts_with("avx512.ptestm") ||
2961 Name.starts_with("avx512.ptestnm")) {
2962 Value *Op0 = CI->getArgOperand(0);
2963 Value *Op1 = CI->getArgOperand(1);
2964 Value *Mask = CI->getArgOperand(2);
2965 Rep = Builder.CreateAnd(Op0, Op1);
2966 llvm::Type *Ty = Op0->getType();
2967 Value *Zero = llvm::Constant::getNullValue(Ty);
2968 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
2969 ? ICmpInst::ICMP_NE
2970 : ICmpInst::ICMP_EQ;
2971 Rep = Builder.CreateICmp(Pred, Rep, Zero);
2972 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
2973 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
2974 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
2975 ->getNumElements();
2976 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
2977 Rep =
2978 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2979 } else if (Name.starts_with("avx512.kunpck")) {
2980 unsigned NumElts = CI->getType()->getScalarSizeInBits();
2981 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
2982 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
2983 int Indices[64];
2984 for (unsigned i = 0; i != NumElts; ++i)
2985 Indices[i] = i;
2986
2987 // First extract half of each vector. This gives better codegen than
2988 // doing it in a single shuffle.
2989 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
2990 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
2991 // Concat the vectors.
2992 // NOTE: Operands have to be swapped to match intrinsic definition.
2993 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
2994 Rep = Builder.CreateBitCast(Rep, CI->getType());
2995 } else if (Name == "avx512.kand.w") {
2996 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2997 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2998 Rep = Builder.CreateAnd(LHS, RHS);
2999 Rep = Builder.CreateBitCast(Rep, CI->getType());
3000 } else if (Name == "avx512.kandn.w") {
3001 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3002 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3003 LHS = Builder.CreateNot(LHS);
3004 Rep = Builder.CreateAnd(LHS, RHS);
3005 Rep = Builder.CreateBitCast(Rep, CI->getType());
3006 } else if (Name == "avx512.kor.w") {
3007 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3008 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3009 Rep = Builder.CreateOr(LHS, RHS);
3010 Rep = Builder.CreateBitCast(Rep, CI->getType());
3011 } else if (Name == "avx512.kxor.w") {
3012 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3013 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3014 Rep = Builder.CreateXor(LHS, RHS);
3015 Rep = Builder.CreateBitCast(Rep, CI->getType());
3016 } else if (Name == "avx512.kxnor.w") {
3017 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3018 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3019 LHS = Builder.CreateNot(LHS);
3020 Rep = Builder.CreateXor(LHS, RHS);
3021 Rep = Builder.CreateBitCast(Rep, CI->getType());
3022 } else if (Name == "avx512.knot.w") {
3023 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3024 Rep = Builder.CreateNot(Rep);
3025 Rep = Builder.CreateBitCast(Rep, CI->getType());
3026 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3027 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3028 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3029 Rep = Builder.CreateOr(LHS, RHS);
3030 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3031 Value *C;
3032 if (Name[14] == 'c')
3033 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3034 else
3035 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3036 Rep = Builder.CreateICmpEQ(Rep, C);
3037 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3038 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3039 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3040 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3041 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3042 Type *I32Ty = Type::getInt32Ty(C);
3043 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3044 ConstantInt::get(I32Ty, 0));
3045 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3046 ConstantInt::get(I32Ty, 0));
3047 Value *EltOp;
3048 if (Name.contains(".add."))
3049 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3050 else if (Name.contains(".sub."))
3051 EltOp = Builder.CreateFSub(Elt0, Elt1);
3052 else if (Name.contains(".mul."))
3053 EltOp = Builder.CreateFMul(Elt0, Elt1);
3054 else
3055 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3056 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3057 ConstantInt::get(I32Ty, 0));
3058 } else if (Name.starts_with("avx512.mask.pcmp")) {
3059 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3060 bool CmpEq = Name[16] == 'e';
3061 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3062 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3063 Type *OpTy = CI->getArgOperand(0)->getType();
3064 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3065 Intrinsic::ID IID;
3066 switch (VecWidth) {
3067 default:
3068 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3069 break;
3070 case 128:
3071 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3072 break;
3073 case 256:
3074 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3075 break;
3076 case 512:
3077 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3078 break;
3079 }
3080
3081 Rep =
3082 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3083 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3084 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3085 Type *OpTy = CI->getArgOperand(0)->getType();
3086 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3087 unsigned EltWidth = OpTy->getScalarSizeInBits();
3088 Intrinsic::ID IID;
3089 if (VecWidth == 128 && EltWidth == 32)
3090 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3091 else if (VecWidth == 256 && EltWidth == 32)
3092 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3093 else if (VecWidth == 512 && EltWidth == 32)
3094 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3095 else if (VecWidth == 128 && EltWidth == 64)
3096 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3097 else if (VecWidth == 256 && EltWidth == 64)
3098 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3099 else if (VecWidth == 512 && EltWidth == 64)
3100 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3101 else
3102 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3103
3104 Rep =
3105 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3106 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3107 } else if (Name.starts_with("avx512.cmp.p")) {
3108 SmallVector<Value *, 4> Args(CI->args());
3109 Type *OpTy = Args[0]->getType();
3110 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3111 unsigned EltWidth = OpTy->getScalarSizeInBits();
3112 Intrinsic::ID IID;
3113 if (VecWidth == 128 && EltWidth == 32)
3114 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3115 else if (VecWidth == 256 && EltWidth == 32)
3116 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3117 else if (VecWidth == 512 && EltWidth == 32)
3118 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3119 else if (VecWidth == 128 && EltWidth == 64)
3120 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3121 else if (VecWidth == 256 && EltWidth == 64)
3122 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3123 else if (VecWidth == 512 && EltWidth == 64)
3124 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3125 else
3126 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3127
3128 Value *Mask = Constant::getAllOnesValue(CI->getType());
3129 if (VecWidth == 512)
3130 std::swap(Mask, Args.back());
3131 Args.push_back(Mask);
3132
3133 Rep = Builder.CreateIntrinsic(IID, Args);
3134 } else if (Name.starts_with("avx512.mask.cmp.")) {
3135 // Integer compare intrinsics.
3136 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3137 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3138 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3139 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3140 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3141 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3142 Name.starts_with("avx512.cvtw2mask.") ||
3143 Name.starts_with("avx512.cvtd2mask.") ||
3144 Name.starts_with("avx512.cvtq2mask.")) {
3145 Value *Op = CI->getArgOperand(0);
3146 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3147 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3148 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3149 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3150 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3151 Name.starts_with("avx512.mask.pabs")) {
3152 Rep = upgradeAbs(Builder, *CI);
3153 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3154 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3155 Name.starts_with("avx512.mask.pmaxs")) {
3156 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3157 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3158 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3159 Name.starts_with("avx512.mask.pmaxu")) {
3160 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3161 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3162 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3163 Name.starts_with("avx512.mask.pmins")) {
3164 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3165 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3166 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3167 Name.starts_with("avx512.mask.pminu")) {
3168 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3169 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3170 Name == "avx512.pmulu.dq.512" ||
3171 Name.starts_with("avx512.mask.pmulu.dq.")) {
3172 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3173 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3174 Name == "avx512.pmul.dq.512" ||
3175 Name.starts_with("avx512.mask.pmul.dq.")) {
3176 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3177 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3178 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3179 Rep =
3180 Builder.CreateSIToFP(CI->getArgOperand(1),
3181 cast<VectorType>(CI->getType())->getElementType());
3182 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3183 } else if (Name == "avx512.cvtusi2sd") {
3184 Rep =
3185 Builder.CreateUIToFP(CI->getArgOperand(1),
3186 cast<VectorType>(CI->getType())->getElementType());
3187 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3188 } else if (Name == "sse2.cvtss2sd") {
3189 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3190 Rep = Builder.CreateFPExt(
3191 Rep, cast<VectorType>(CI->getType())->getElementType());
3192 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3193 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3194 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3195 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3196 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3197 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3198 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3199 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3200 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3201 Name == "avx512.mask.cvtqq2ps.256" ||
3202 Name == "avx512.mask.cvtqq2ps.512" ||
3203 Name == "avx512.mask.cvtuqq2ps.256" ||
3204 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3205 Name == "avx.cvt.ps2.pd.256" ||
3206 Name == "avx512.mask.cvtps2pd.128" ||
3207 Name == "avx512.mask.cvtps2pd.256") {
3208 auto *DstTy = cast<FixedVectorType>(CI->getType());
3209 Rep = CI->getArgOperand(0);
3210 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3211
3212 unsigned NumDstElts = DstTy->getNumElements();
3213 if (NumDstElts < SrcTy->getNumElements()) {
3214 assert(NumDstElts == 2 && "Unexpected vector size");
3215 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3216 }
3217
3218 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3219 bool IsUnsigned = Name.contains("cvtu");
3220 if (IsPS2PD)
3221 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3222 else if (CI->arg_size() == 4 &&
3223 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3224 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3225 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3226 : Intrinsic::x86_avx512_sitofp_round;
3227 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3228 {Rep, CI->getArgOperand(3)});
3229 } else {
3230 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3231 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3232 }
3233
3234 if (CI->arg_size() >= 3)
3235 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3236 CI->getArgOperand(1));
3237 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3238 Name.starts_with("vcvtph2ps.")) {
3239 auto *DstTy = cast<FixedVectorType>(CI->getType());
3240 Rep = CI->getArgOperand(0);
3241 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3242 unsigned NumDstElts = DstTy->getNumElements();
3243 if (NumDstElts != SrcTy->getNumElements()) {
3244 assert(NumDstElts == 4 && "Unexpected vector size");
3245 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3246 }
3247 Rep = Builder.CreateBitCast(
3248 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3249 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3250 if (CI->arg_size() >= 3)
3251 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3252 CI->getArgOperand(1));
3253 } else if (Name.starts_with("avx512.mask.load")) {
3254 // "avx512.mask.loadu." or "avx512.mask.load."
3255 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3256 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3257 CI->getArgOperand(2), Aligned);
3258 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3259 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3260 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3261 ResultTy->getNumElements());
3262
3263 Rep = Builder.CreateIntrinsic(
3264 Intrinsic::masked_expandload, ResultTy,
3265 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3266 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3267 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3268 Value *MaskVec =
3269 getX86MaskVec(Builder, CI->getArgOperand(2),
3270 cast<FixedVectorType>(ResultTy)->getNumElements());
3271
3272 Rep = Builder.CreateIntrinsic(
3273 Intrinsic::masked_compressstore, ResultTy,
3274 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3275 } else if (Name.starts_with("avx512.mask.compress.") ||
3276 Name.starts_with("avx512.mask.expand.")) {
3277 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3278
3279 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3280 ResultTy->getNumElements());
3281
3282 bool IsCompress = Name[12] == 'c';
3283 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3284 : Intrinsic::x86_avx512_mask_expand;
3285 Rep = Builder.CreateIntrinsic(
3286 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3287 } else if (Name.starts_with("xop.vpcom")) {
3288 bool IsSigned;
3289 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3290 Name.ends_with("uq"))
3291 IsSigned = false;
3292 else if (Name.ends_with("b") || Name.ends_with("w") ||
3293 Name.ends_with("d") || Name.ends_with("q"))
3294 IsSigned = true;
3295 else
3296 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3297
3298 unsigned Imm;
3299 if (CI->arg_size() == 3) {
3300 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3301 } else {
3302 Name = Name.substr(9); // strip off "xop.vpcom"
3303 if (Name.starts_with("lt"))
3304 Imm = 0;
3305 else if (Name.starts_with("le"))
3306 Imm = 1;
3307 else if (Name.starts_with("gt"))
3308 Imm = 2;
3309 else if (Name.starts_with("ge"))
3310 Imm = 3;
3311 else if (Name.starts_with("eq"))
3312 Imm = 4;
3313 else if (Name.starts_with("ne"))
3314 Imm = 5;
3315 else if (Name.starts_with("false"))
3316 Imm = 6;
3317 else if (Name.starts_with("true"))
3318 Imm = 7;
3319 else
3320 llvm_unreachable("Unknown condition");
3321 }
3322
3323 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3324 } else if (Name.starts_with("xop.vpcmov")) {
3325 Value *Sel = CI->getArgOperand(2);
3326 Value *NotSel = Builder.CreateNot(Sel);
3327 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3328 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3329 Rep = Builder.CreateOr(Sel0, Sel1);
3330 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3331 Name.starts_with("avx512.mask.prol")) {
3332 Rep = upgradeX86Rotate(Builder, *CI, false);
3333 } else if (Name.starts_with("avx512.pror") ||
3334 Name.starts_with("avx512.mask.pror")) {
3335 Rep = upgradeX86Rotate(Builder, *CI, true);
3336 } else if (Name.starts_with("avx512.vpshld.") ||
3337 Name.starts_with("avx512.mask.vpshld") ||
3338 Name.starts_with("avx512.maskz.vpshld")) {
3339 bool ZeroMask = Name[11] == 'z';
3340 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3341 } else if (Name.starts_with("avx512.vpshrd.") ||
3342 Name.starts_with("avx512.mask.vpshrd") ||
3343 Name.starts_with("avx512.maskz.vpshrd")) {
3344 bool ZeroMask = Name[11] == 'z';
3345 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3346 } else if (Name == "sse42.crc32.64.8") {
3347 Value *Trunc0 =
3348 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3349 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3350 {Trunc0, CI->getArgOperand(1)});
3351 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3352 } else if (Name.starts_with("avx.vbroadcast.s") ||
3353 Name.starts_with("avx512.vbroadcast.s")) {
3354 // Replace broadcasts with a series of insertelements.
3355 auto *VecTy = cast<FixedVectorType>(CI->getType());
3356 Type *EltTy = VecTy->getElementType();
3357 unsigned EltNum = VecTy->getNumElements();
3358 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3359 Type *I32Ty = Type::getInt32Ty(C);
3360 Rep = PoisonValue::get(VecTy);
3361 for (unsigned I = 0; I < EltNum; ++I)
3362 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3363 } else if (Name.starts_with("sse41.pmovsx") ||
3364 Name.starts_with("sse41.pmovzx") ||
3365 Name.starts_with("avx2.pmovsx") ||
3366 Name.starts_with("avx2.pmovzx") ||
3367 Name.starts_with("avx512.mask.pmovsx") ||
3368 Name.starts_with("avx512.mask.pmovzx")) {
3369 auto *DstTy = cast<FixedVectorType>(CI->getType());
3370 unsigned NumDstElts = DstTy->getNumElements();
3371
3372 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3373 SmallVector<int, 8> ShuffleMask(NumDstElts);
3374 for (unsigned i = 0; i != NumDstElts; ++i)
3375 ShuffleMask[i] = i;
3376
3377 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3378
3379 bool DoSext = Name.contains("pmovsx");
3380 Rep =
3381 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3382 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3383 if (CI->arg_size() == 3)
3384 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3385 CI->getArgOperand(1));
3386 } else if (Name == "avx512.mask.pmov.qd.256" ||
3387 Name == "avx512.mask.pmov.qd.512" ||
3388 Name == "avx512.mask.pmov.wb.256" ||
3389 Name == "avx512.mask.pmov.wb.512") {
3390 Type *Ty = CI->getArgOperand(1)->getType();
3391 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3392 Rep =
3393 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3394 } else if (Name.starts_with("avx.vbroadcastf128") ||
3395 Name == "avx2.vbroadcasti128") {
3396 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3397 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3398 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3399 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3400 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3401 if (NumSrcElts == 2)
3402 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3403 else
3404 Rep = Builder.CreateShuffleVector(Load,
3405 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3406 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3407 Name.starts_with("avx512.mask.shuf.f")) {
3408 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3409 Type *VT = CI->getType();
3410 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3411 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3412 unsigned ControlBitsMask = NumLanes - 1;
3413 unsigned NumControlBits = NumLanes / 2;
3414 SmallVector<int, 8> ShuffleMask(0);
3415
3416 for (unsigned l = 0; l != NumLanes; ++l) {
3417 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3418 // We actually need the other source.
3419 if (l >= NumLanes / 2)
3420 LaneMask += NumLanes;
3421 for (unsigned i = 0; i != NumElementsInLane; ++i)
3422 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3423 }
3424 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3425 CI->getArgOperand(1), ShuffleMask);
3426 Rep =
3427 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3428 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3429 Name.starts_with("avx512.mask.broadcasti")) {
3430 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3431 ->getNumElements();
3432 unsigned NumDstElts =
3433 cast<FixedVectorType>(CI->getType())->getNumElements();
3434
3435 SmallVector<int, 8> ShuffleMask(NumDstElts);
3436 for (unsigned i = 0; i != NumDstElts; ++i)
3437 ShuffleMask[i] = i % NumSrcElts;
3438
3439 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3440 CI->getArgOperand(0), ShuffleMask);
3441 Rep =
3442 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3443 } else if (Name.starts_with("avx2.pbroadcast") ||
3444 Name.starts_with("avx2.vbroadcast") ||
3445 Name.starts_with("avx512.pbroadcast") ||
3446 Name.starts_with("avx512.mask.broadcast.s")) {
3447 // Replace vp?broadcasts with a vector shuffle.
3448 Value *Op = CI->getArgOperand(0);
3449 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3450 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3451 SmallVector<int, 8> M;
3452 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3453 Rep = Builder.CreateShuffleVector(Op, M);
3454
3455 if (CI->arg_size() == 3)
3456 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3457 CI->getArgOperand(1));
3458 } else if (Name.starts_with("sse2.padds.") ||
3459 Name.starts_with("avx2.padds.") ||
3460 Name.starts_with("avx512.padds.") ||
3461 Name.starts_with("avx512.mask.padds.")) {
3462 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3463 } else if (Name.starts_with("sse2.psubs.") ||
3464 Name.starts_with("avx2.psubs.") ||
3465 Name.starts_with("avx512.psubs.") ||
3466 Name.starts_with("avx512.mask.psubs.")) {
3467 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3468 } else if (Name.starts_with("sse2.paddus.") ||
3469 Name.starts_with("avx2.paddus.") ||
3470 Name.starts_with("avx512.mask.paddus.")) {
3471 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3472 } else if (Name.starts_with("sse2.psubus.") ||
3473 Name.starts_with("avx2.psubus.") ||
3474 Name.starts_with("avx512.mask.psubus.")) {
3475 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3476 } else if (Name.starts_with("avx512.mask.palignr.")) {
3477 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3478 CI->getArgOperand(1), CI->getArgOperand(2),
3479 CI->getArgOperand(3), CI->getArgOperand(4),
3480 false);
3481 } else if (Name.starts_with("avx512.mask.valign.")) {
3482 Rep = upgradeX86ALIGNIntrinsics(
3483 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3484 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3485 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3486 // 128/256-bit shift left specified in bits.
3487 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3488 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3489 Shift / 8); // Shift is in bits.
3490 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3491 // 128/256-bit shift right specified in bits.
3492 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3493 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3494 Shift / 8); // Shift is in bits.
3495 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3496 Name == "avx512.psll.dq.512") {
3497 // 128/256/512-bit shift left specified in bytes.
3498 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3499 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3500 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3501 Name == "avx512.psrl.dq.512") {
3502 // 128/256/512-bit shift right specified in bytes.
3503 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3504 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3505 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3506 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3507 Name.starts_with("avx2.pblendd.")) {
3508 Value *Op0 = CI->getArgOperand(0);
3509 Value *Op1 = CI->getArgOperand(1);
3510 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3511 auto *VecTy = cast<FixedVectorType>(CI->getType());
3512 unsigned NumElts = VecTy->getNumElements();
3513
3514 SmallVector<int, 16> Idxs(NumElts);
3515 for (unsigned i = 0; i != NumElts; ++i)
3516 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3517
3518 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3519 } else if (Name.starts_with("avx.vinsertf128.") ||
3520 Name == "avx2.vinserti128" ||
3521 Name.starts_with("avx512.mask.insert")) {
3522 Value *Op0 = CI->getArgOperand(0);
3523 Value *Op1 = CI->getArgOperand(1);
3524 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3525 unsigned DstNumElts =
3526 cast<FixedVectorType>(CI->getType())->getNumElements();
3527 unsigned SrcNumElts =
3528 cast<FixedVectorType>(Op1->getType())->getNumElements();
3529 unsigned Scale = DstNumElts / SrcNumElts;
3530
3531 // Mask off the high bits of the immediate value; hardware ignores those.
3532 Imm = Imm % Scale;
3533
3534 // Extend the second operand into a vector the size of the destination.
3535 SmallVector<int, 8> Idxs(DstNumElts);
3536 for (unsigned i = 0; i != SrcNumElts; ++i)
3537 Idxs[i] = i;
3538 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3539 Idxs[i] = SrcNumElts;
3540 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3541
3542 // Insert the second operand into the first operand.
3543
3544 // Note that there is no guarantee that instruction lowering will actually
3545 // produce a vinsertf128 instruction for the created shuffles. In
3546 // particular, the 0 immediate case involves no lane changes, so it can
3547 // be handled as a blend.
3548
3549 // Example of shuffle mask for 32-bit elements:
3550 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3551 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3552
3553 // First fill with identify mask.
3554 for (unsigned i = 0; i != DstNumElts; ++i)
3555 Idxs[i] = i;
3556 // Then replace the elements where we need to insert.
3557 for (unsigned i = 0; i != SrcNumElts; ++i)
3558 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3559 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3560
3561 // If the intrinsic has a mask operand, handle that.
3562 if (CI->arg_size() == 5)
3563 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3564 CI->getArgOperand(3));
3565 } else if (Name.starts_with("avx.vextractf128.") ||
3566 Name == "avx2.vextracti128" ||
3567 Name.starts_with("avx512.mask.vextract")) {
3568 Value *Op0 = CI->getArgOperand(0);
3569 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3570 unsigned DstNumElts =
3571 cast<FixedVectorType>(CI->getType())->getNumElements();
3572 unsigned SrcNumElts =
3573 cast<FixedVectorType>(Op0->getType())->getNumElements();
3574 unsigned Scale = SrcNumElts / DstNumElts;
3575
3576 // Mask off the high bits of the immediate value; hardware ignores those.
3577 Imm = Imm % Scale;
3578
3579 // Get indexes for the subvector of the input vector.
3580 SmallVector<int, 8> Idxs(DstNumElts);
3581 for (unsigned i = 0; i != DstNumElts; ++i) {
3582 Idxs[i] = i + (Imm * DstNumElts);
3583 }
3584 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3585
3586 // If the intrinsic has a mask operand, handle that.
3587 if (CI->arg_size() == 4)
3588 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3589 CI->getArgOperand(2));
3590 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3591 Name.starts_with("avx512.mask.perm.di.")) {
3592 Value *Op0 = CI->getArgOperand(0);
3593 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3594 auto *VecTy = cast<FixedVectorType>(CI->getType());
3595 unsigned NumElts = VecTy->getNumElements();
3596
3597 SmallVector<int, 8> Idxs(NumElts);
3598 for (unsigned i = 0; i != NumElts; ++i)
3599 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3600
3601 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3602
3603 if (CI->arg_size() == 4)
3604 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3605 CI->getArgOperand(2));
3606 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3607 // The immediate permute control byte looks like this:
3608 // [1:0] - select 128 bits from sources for low half of destination
3609 // [2] - ignore
3610 // [3] - zero low half of destination
3611 // [5:4] - select 128 bits from sources for high half of destination
3612 // [6] - ignore
3613 // [7] - zero high half of destination
3614
3615 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3616
3617 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3618 unsigned HalfSize = NumElts / 2;
3619 SmallVector<int, 8> ShuffleMask(NumElts);
3620
3621 // Determine which operand(s) are actually in use for this instruction.
3622 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3623 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3624
3625 // If needed, replace operands based on zero mask.
3626 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3627 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3628
3629 // Permute low half of result.
3630 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3631 for (unsigned i = 0; i < HalfSize; ++i)
3632 ShuffleMask[i] = StartIndex + i;
3633
3634 // Permute high half of result.
3635 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3636 for (unsigned i = 0; i < HalfSize; ++i)
3637 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3638
3639 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3640
3641 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
3642 Name.starts_with("avx512.mask.vpermil.p") ||
3643 Name.starts_with("avx512.mask.pshuf.d.")) {
3644 Value *Op0 = CI->getArgOperand(0);
3645 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3646 auto *VecTy = cast<FixedVectorType>(CI->getType());
3647 unsigned NumElts = VecTy->getNumElements();
3648 // Calculate the size of each index in the immediate.
3649 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3650 unsigned IdxMask = ((1 << IdxSize) - 1);
3651
3652 SmallVector<int, 8> Idxs(NumElts);
3653 // Lookup the bits for this element, wrapping around the immediate every
3654 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3655 // to offset by the first index of each group.
3656 for (unsigned i = 0; i != NumElts; ++i)
3657 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3658
3659 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3660
3661 if (CI->arg_size() == 4)
3662 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3663 CI->getArgOperand(2));
3664 } else if (Name == "sse2.pshufl.w" ||
3665 Name.starts_with("avx512.mask.pshufl.w.")) {
3666 Value *Op0 = CI->getArgOperand(0);
3667 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3668 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3669
3670 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
3671 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3672
3673 SmallVector<int, 16> Idxs(NumElts);
3674 for (unsigned l = 0; l != NumElts; l += 8) {
3675 for (unsigned i = 0; i != 4; ++i)
3676 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3677 for (unsigned i = 4; i != 8; ++i)
3678 Idxs[i + l] = i + l;
3679 }
3680
3681 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3682
3683 if (CI->arg_size() == 4)
3684 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3685 CI->getArgOperand(2));
3686 } else if (Name == "sse2.pshufh.w" ||
3687 Name.starts_with("avx512.mask.pshufh.w.")) {
3688 Value *Op0 = CI->getArgOperand(0);
3689 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3690 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3691
3692 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
3693 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3694
3695 SmallVector<int, 16> Idxs(NumElts);
3696 for (unsigned l = 0; l != NumElts; l += 8) {
3697 for (unsigned i = 0; i != 4; ++i)
3698 Idxs[i + l] = i + l;
3699 for (unsigned i = 0; i != 4; ++i)
3700 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3701 }
3702
3703 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3704
3705 if (CI->arg_size() == 4)
3706 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3707 CI->getArgOperand(2));
3708 } else if (Name.starts_with("avx512.mask.shuf.p")) {
3709 Value *Op0 = CI->getArgOperand(0);
3710 Value *Op1 = CI->getArgOperand(1);
3711 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3712 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3713
3714 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3715 unsigned HalfLaneElts = NumLaneElts / 2;
3716
3717 SmallVector<int, 16> Idxs(NumElts);
3718 for (unsigned i = 0; i != NumElts; ++i) {
3719 // Base index is the starting element of the lane.
3720 Idxs[i] = i - (i % NumLaneElts);
3721 // If we are half way through the lane switch to the other source.
3722 if ((i % NumLaneElts) >= HalfLaneElts)
3723 Idxs[i] += NumElts;
3724 // Now select the specific element. By adding HalfLaneElts bits from
3725 // the immediate. Wrapping around the immediate every 8-bits.
3726 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3727 }
3728
3729 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3730
3731 Rep =
3732 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3733 } else if (Name.starts_with("avx512.mask.movddup") ||
3734 Name.starts_with("avx512.mask.movshdup") ||
3735 Name.starts_with("avx512.mask.movsldup")) {
3736 Value *Op0 = CI->getArgOperand(0);
3737 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3738 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3739
3740 unsigned Offset = 0;
3741 if (Name.starts_with("avx512.mask.movshdup."))
3742 Offset = 1;
3743
3744 SmallVector<int, 16> Idxs(NumElts);
3745 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3746 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3747 Idxs[i + l + 0] = i + l + Offset;
3748 Idxs[i + l + 1] = i + l + Offset;
3749 }
3750
3751 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3752
3753 Rep =
3754 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3755 } else if (Name.starts_with("avx512.mask.punpckl") ||
3756 Name.starts_with("avx512.mask.unpckl.")) {
3757 Value *Op0 = CI->getArgOperand(0);
3758 Value *Op1 = CI->getArgOperand(1);
3759 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3760 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3761
3762 SmallVector<int, 64> Idxs(NumElts);
3763 for (int l = 0; l != NumElts; l += NumLaneElts)
3764 for (int i = 0; i != NumLaneElts; ++i)
3765 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3766
3767 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3768
3769 Rep =
3770 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3771 } else if (Name.starts_with("avx512.mask.punpckh") ||
3772 Name.starts_with("avx512.mask.unpckh.")) {
3773 Value *Op0 = CI->getArgOperand(0);
3774 Value *Op1 = CI->getArgOperand(1);
3775 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3776 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3777
3778 SmallVector<int, 64> Idxs(NumElts);
3779 for (int l = 0; l != NumElts; l += NumLaneElts)
3780 for (int i = 0; i != NumLaneElts; ++i)
3781 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3782
3783 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3784
3785 Rep =
3786 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3787 } else if (Name.starts_with("avx512.mask.and.") ||
3788 Name.starts_with("avx512.mask.pand.")) {
3789 VectorType *FTy = cast<VectorType>(CI->getType());
3790 VectorType *ITy = VectorType::getInteger(FTy);
3791 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3792 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3793 Rep = Builder.CreateBitCast(Rep, FTy);
3794 Rep =
3795 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3796 } else if (Name.starts_with("avx512.mask.andn.") ||
3797 Name.starts_with("avx512.mask.pandn.")) {
3798 VectorType *FTy = cast<VectorType>(CI->getType());
3799 VectorType *ITy = VectorType::getInteger(FTy);
3800 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3801 Rep = Builder.CreateAnd(Rep,
3802 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3803 Rep = Builder.CreateBitCast(Rep, FTy);
3804 Rep =
3805 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3806 } else if (Name.starts_with("avx512.mask.or.") ||
3807 Name.starts_with("avx512.mask.por.")) {
3808 VectorType *FTy = cast<VectorType>(CI->getType());
3809 VectorType *ITy = VectorType::getInteger(FTy);
3810 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3811 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3812 Rep = Builder.CreateBitCast(Rep, FTy);
3813 Rep =
3814 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3815 } else if (Name.starts_with("avx512.mask.xor.") ||
3816 Name.starts_with("avx512.mask.pxor.")) {
3817 VectorType *FTy = cast<VectorType>(CI->getType());
3818 VectorType *ITy = VectorType::getInteger(FTy);
3819 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3820 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3821 Rep = Builder.CreateBitCast(Rep, FTy);
3822 Rep =
3823 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3824 } else if (Name.starts_with("avx512.mask.padd.")) {
3825 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3826 Rep =
3827 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3828 } else if (Name.starts_with("avx512.mask.psub.")) {
3829 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3830 Rep =
3831 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3832 } else if (Name.starts_with("avx512.mask.pmull.")) {
3833 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3834 Rep =
3835 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3836 } else if (Name.starts_with("avx512.mask.add.p")) {
3837 if (Name.ends_with(".512")) {
3838 Intrinsic::ID IID;
3839 if (Name[17] == 's')
3840 IID = Intrinsic::x86_avx512_add_ps_512;
3841 else
3842 IID = Intrinsic::x86_avx512_add_pd_512;
3843
3844 Rep = Builder.CreateIntrinsic(
3845 IID,
3846 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3847 } else {
3848 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3849 }
3850 Rep =
3851 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3852 } else if (Name.starts_with("avx512.mask.div.p")) {
3853 if (Name.ends_with(".512")) {
3854 Intrinsic::ID IID;
3855 if (Name[17] == 's')
3856 IID = Intrinsic::x86_avx512_div_ps_512;
3857 else
3858 IID = Intrinsic::x86_avx512_div_pd_512;
3859
3860 Rep = Builder.CreateIntrinsic(
3861 IID,
3862 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3863 } else {
3864 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3865 }
3866 Rep =
3867 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3868 } else if (Name.starts_with("avx512.mask.mul.p")) {
3869 if (Name.ends_with(".512")) {
3870 Intrinsic::ID IID;
3871 if (Name[17] == 's')
3872 IID = Intrinsic::x86_avx512_mul_ps_512;
3873 else
3874 IID = Intrinsic::x86_avx512_mul_pd_512;
3875
3876 Rep = Builder.CreateIntrinsic(
3877 IID,
3878 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3879 } else {
3880 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
3881 }
3882 Rep =
3883 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3884 } else if (Name.starts_with("avx512.mask.sub.p")) {
3885 if (Name.ends_with(".512")) {
3886 Intrinsic::ID IID;
3887 if (Name[17] == 's')
3888 IID = Intrinsic::x86_avx512_sub_ps_512;
3889 else
3890 IID = Intrinsic::x86_avx512_sub_pd_512;
3891
3892 Rep = Builder.CreateIntrinsic(
3893 IID,
3894 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3895 } else {
3896 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
3897 }
3898 Rep =
3899 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3900 } else if ((Name.starts_with("avx512.mask.max.p") ||
3901 Name.starts_with("avx512.mask.min.p")) &&
3902 Name.drop_front(18) == ".512") {
3903 bool IsDouble = Name[17] == 'd';
3904 bool IsMin = Name[13] == 'i';
3905 static const Intrinsic::ID MinMaxTbl[2][2] = {
3906 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
3907 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
3908 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
3909
3910 Rep = Builder.CreateIntrinsic(
3911 IID,
3912 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3913 Rep =
3914 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3915 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
3916 Rep =
3917 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
3918 {CI->getArgOperand(0), Builder.getInt1(false)});
3919 Rep =
3920 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3921 } else if (Name.starts_with("avx512.mask.psll")) {
3922 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
3923 bool IsVariable = Name[16] == 'v';
3924 char Size = Name[16] == '.' ? Name[17]
3925 : Name[17] == '.' ? Name[18]
3926 : Name[18] == '.' ? Name[19]
3927 : Name[20];
3928
3929 Intrinsic::ID IID;
3930 if (IsVariable && Name[17] != '.') {
3931 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
3932 IID = Intrinsic::x86_avx2_psllv_q;
3933 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
3934 IID = Intrinsic::x86_avx2_psllv_q_256;
3935 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
3936 IID = Intrinsic::x86_avx2_psllv_d;
3937 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3938 IID = Intrinsic::x86_avx2_psllv_d_256;
3939 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3940 IID = Intrinsic::x86_avx512_psllv_w_128;
3941 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3942 IID = Intrinsic::x86_avx512_psllv_w_256;
3943 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3944 IID = Intrinsic::x86_avx512_psllv_w_512;
3945 else
3946 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
3947 } else if (Name.ends_with(".128")) {
3948 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3949 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3950 : Intrinsic::x86_sse2_psll_d;
3951 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3952 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3953 : Intrinsic::x86_sse2_psll_q;
3954 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3955 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3956 : Intrinsic::x86_sse2_psll_w;
3957 else
3958 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
3959 } else if (Name.ends_with(".256")) {
3960 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3961 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3962 : Intrinsic::x86_avx2_psll_d;
3963 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3964 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3965 : Intrinsic::x86_avx2_psll_q;
3966 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3967 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3968 : Intrinsic::x86_avx2_psll_w;
3969 else
3970 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
3971 } else {
3972 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3973 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
3974 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
3975 : Intrinsic::x86_avx512_psll_d_512;
3976 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3977 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
3978 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
3979 : Intrinsic::x86_avx512_psll_q_512;
3980 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3981 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3982 : Intrinsic::x86_avx512_psll_w_512;
3983 else
3984 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
3985 }
3986
3987 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
3988 } else if (Name.starts_with("avx512.mask.psrl")) {
3989 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
3990 bool IsVariable = Name[16] == 'v';
3991 char Size = Name[16] == '.' ? Name[17]
3992 : Name[17] == '.' ? Name[18]
3993 : Name[18] == '.' ? Name[19]
3994 : Name[20];
3995
3996 Intrinsic::ID IID;
3997 if (IsVariable && Name[17] != '.') {
3998 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3999 IID = Intrinsic::x86_avx2_psrlv_q;
4000 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4001 IID = Intrinsic::x86_avx2_psrlv_q_256;
4002 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4003 IID = Intrinsic::x86_avx2_psrlv_d;
4004 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4005 IID = Intrinsic::x86_avx2_psrlv_d_256;
4006 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4007 IID = Intrinsic::x86_avx512_psrlv_w_128;
4008 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4009 IID = Intrinsic::x86_avx512_psrlv_w_256;
4010 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4011 IID = Intrinsic::x86_avx512_psrlv_w_512;
4012 else
4013 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4014 } else if (Name.ends_with(".128")) {
4015 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4016 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4017 : Intrinsic::x86_sse2_psrl_d;
4018 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4019 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4020 : Intrinsic::x86_sse2_psrl_q;
4021 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4022 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4023 : Intrinsic::x86_sse2_psrl_w;
4024 else
4025 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4026 } else if (Name.ends_with(".256")) {
4027 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4028 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4029 : Intrinsic::x86_avx2_psrl_d;
4030 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4031 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4032 : Intrinsic::x86_avx2_psrl_q;
4033 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4034 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4035 : Intrinsic::x86_avx2_psrl_w;
4036 else
4037 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4038 } else {
4039 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4040 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4041 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4042 : Intrinsic::x86_avx512_psrl_d_512;
4043 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4044 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4045 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4046 : Intrinsic::x86_avx512_psrl_q_512;
4047 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4048 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4049 : Intrinsic::x86_avx512_psrl_w_512;
4050 else
4051 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4052 }
4053
4054 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4055 } else if (Name.starts_with("avx512.mask.psra")) {
4056 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4057 bool IsVariable = Name[16] == 'v';
4058 char Size = Name[16] == '.' ? Name[17]
4059 : Name[17] == '.' ? Name[18]
4060 : Name[18] == '.' ? Name[19]
4061 : Name[20];
4062
4063 Intrinsic::ID IID;
4064 if (IsVariable && Name[17] != '.') {
4065 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4066 IID = Intrinsic::x86_avx2_psrav_d;
4067 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4068 IID = Intrinsic::x86_avx2_psrav_d_256;
4069 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4070 IID = Intrinsic::x86_avx512_psrav_w_128;
4071 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4072 IID = Intrinsic::x86_avx512_psrav_w_256;
4073 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4074 IID = Intrinsic::x86_avx512_psrav_w_512;
4075 else
4076 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4077 } else if (Name.ends_with(".128")) {
4078 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4079 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4080 : Intrinsic::x86_sse2_psra_d;
4081 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4082 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4083 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4084 : Intrinsic::x86_avx512_psra_q_128;
4085 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4086 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4087 : Intrinsic::x86_sse2_psra_w;
4088 else
4089 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4090 } else if (Name.ends_with(".256")) {
4091 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4092 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4093 : Intrinsic::x86_avx2_psra_d;
4094 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4095 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4096 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4097 : Intrinsic::x86_avx512_psra_q_256;
4098 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4099 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4100 : Intrinsic::x86_avx2_psra_w;
4101 else
4102 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4103 } else {
4104 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4105 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4106 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4107 : Intrinsic::x86_avx512_psra_d_512;
4108 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4109 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4110 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4111 : Intrinsic::x86_avx512_psra_q_512;
4112 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4113 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4114 : Intrinsic::x86_avx512_psra_w_512;
4115 else
4116 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4117 }
4118
4119 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4120 } else if (Name.starts_with("avx512.mask.move.s")) {
4121 Rep = upgradeMaskedMove(Builder, *CI);
4122 } else if (Name.starts_with("avx512.cvtmask2")) {
4123 Rep = upgradeMaskToInt(Builder, *CI);
4124 } else if (Name.ends_with(".movntdqa")) {
4125 MDNode *Node = MDNode::get(
4126 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4127
4128 LoadInst *LI = Builder.CreateAlignedLoad(
4129 CI->getType(), CI->getArgOperand(0),
4130 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4131 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4132 Rep = LI;
4133 } else if (Name.starts_with("fma.vfmadd.") ||
4134 Name.starts_with("fma.vfmsub.") ||
4135 Name.starts_with("fma.vfnmadd.") ||
4136 Name.starts_with("fma.vfnmsub.")) {
4137 bool NegMul = Name[6] == 'n';
4138 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4139 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4140
4141 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4142 CI->getArgOperand(2)};
4143
4144 if (IsScalar) {
4145 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4146 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4147 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4148 }
4149
4150 if (NegMul && !IsScalar)
4151 Ops[0] = Builder.CreateFNeg(Ops[0]);
4152 if (NegMul && IsScalar)
4153 Ops[1] = Builder.CreateFNeg(Ops[1]);
4154 if (NegAcc)
4155 Ops[2] = Builder.CreateFNeg(Ops[2]);
4156
4157 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4158
4159 if (IsScalar)
4160 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4161 } else if (Name.starts_with("fma4.vfmadd.s")) {
4162 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4163 CI->getArgOperand(2)};
4164
4165 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4166 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4167 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4168
4169 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4170
4171 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4172 Rep, (uint64_t)0);
4173 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4174 Name.starts_with("avx512.maskz.vfmadd.s") ||
4175 Name.starts_with("avx512.mask3.vfmadd.s") ||
4176 Name.starts_with("avx512.mask3.vfmsub.s") ||
4177 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4178 bool IsMask3 = Name[11] == '3';
4179 bool IsMaskZ = Name[11] == 'z';
4180 // Drop the "avx512.mask." to make it easier.
4181 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4182 bool NegMul = Name[2] == 'n';
4183 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4184
4185 Value *A = CI->getArgOperand(0);
4186 Value *B = CI->getArgOperand(1);
4187 Value *C = CI->getArgOperand(2);
4188
4189 if (NegMul && (IsMask3 || IsMaskZ))
4190 A = Builder.CreateFNeg(A);
4191 if (NegMul && !(IsMask3 || IsMaskZ))
4192 B = Builder.CreateFNeg(B);
4193 if (NegAcc)
4194 C = Builder.CreateFNeg(C);
4195
4196 A = Builder.CreateExtractElement(A, (uint64_t)0);
4197 B = Builder.CreateExtractElement(B, (uint64_t)0);
4198 C = Builder.CreateExtractElement(C, (uint64_t)0);
4199
4200 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4201 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4202 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4203
4204 Intrinsic::ID IID;
4205 if (Name.back() == 'd')
4206 IID = Intrinsic::x86_avx512_vfmadd_f64;
4207 else
4208 IID = Intrinsic::x86_avx512_vfmadd_f32;
4209 Rep = Builder.CreateIntrinsic(IID, Ops);
4210 } else {
4211 Rep = Builder.CreateFMA(A, B, C);
4212 }
4213
4214 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4215 : IsMask3 ? C
4216 : A;
4217
4218 // For Mask3 with NegAcc, we need to create a new extractelement that
4219 // avoids the negation above.
4220 if (NegAcc && IsMask3)
4221 PassThru =
4222 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4223
4224 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4225 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4226 (uint64_t)0);
4227 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4228 Name.starts_with("avx512.mask.vfnmadd.p") ||
4229 Name.starts_with("avx512.mask.vfnmsub.p") ||
4230 Name.starts_with("avx512.mask3.vfmadd.p") ||
4231 Name.starts_with("avx512.mask3.vfmsub.p") ||
4232 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4233 Name.starts_with("avx512.maskz.vfmadd.p")) {
4234 bool IsMask3 = Name[11] == '3';
4235 bool IsMaskZ = Name[11] == 'z';
4236 // Drop the "avx512.mask." to make it easier.
4237 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4238 bool NegMul = Name[2] == 'n';
4239 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4240
4241 Value *A = CI->getArgOperand(0);
4242 Value *B = CI->getArgOperand(1);
4243 Value *C = CI->getArgOperand(2);
4244
4245 if (NegMul && (IsMask3 || IsMaskZ))
4246 A = Builder.CreateFNeg(A);
4247 if (NegMul && !(IsMask3 || IsMaskZ))
4248 B = Builder.CreateFNeg(B);
4249 if (NegAcc)
4250 C = Builder.CreateFNeg(C);
4251
4252 if (CI->arg_size() == 5 &&
4253 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4254 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4255 Intrinsic::ID IID;
4256 // Check the character before ".512" in string.
4257 if (Name[Name.size() - 5] == 's')
4258 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4259 else
4260 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4261
4262 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4263 } else {
4264 Rep = Builder.CreateFMA(A, B, C);
4265 }
4266
4267 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4268 : IsMask3 ? CI->getArgOperand(2)
4269 : CI->getArgOperand(0);
4270
4271 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4272 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4273 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4274 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4275 Intrinsic::ID IID;
4276 if (VecWidth == 128 && EltWidth == 32)
4277 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4278 else if (VecWidth == 256 && EltWidth == 32)
4279 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4280 else if (VecWidth == 128 && EltWidth == 64)
4281 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4282 else if (VecWidth == 256 && EltWidth == 64)
4283 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4284 else
4285 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4286
4287 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4288 CI->getArgOperand(2)};
4289 Ops[2] = Builder.CreateFNeg(Ops[2]);
4290 Rep = Builder.CreateIntrinsic(IID, Ops);
4291 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4292 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4293 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4294 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4295 bool IsMask3 = Name[11] == '3';
4296 bool IsMaskZ = Name[11] == 'z';
4297 // Drop the "avx512.mask." to make it easier.
4298 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4299 bool IsSubAdd = Name[3] == 's';
4300 if (CI->arg_size() == 5) {
4301 Intrinsic::ID IID;
4302 // Check the character before ".512" in string.
4303 if (Name[Name.size() - 5] == 's')
4304 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4305 else
4306 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4307
4308 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4309 CI->getArgOperand(2), CI->getArgOperand(4)};
4310 if (IsSubAdd)
4311 Ops[2] = Builder.CreateFNeg(Ops[2]);
4312
4313 Rep = Builder.CreateIntrinsic(IID, Ops);
4314 } else {
4315 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4316
4317 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4318 CI->getArgOperand(2)};
4319
4320 Function *FMA = Intrinsic::getOrInsertDeclaration(
4321 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4322 Value *Odd = Builder.CreateCall(FMA, Ops);
4323 Ops[2] = Builder.CreateFNeg(Ops[2]);
4324 Value *Even = Builder.CreateCall(FMA, Ops);
4325
4326 if (IsSubAdd)
4327 std::swap(Even, Odd);
4328
4329 SmallVector<int, 32> Idxs(NumElts);
4330 for (int i = 0; i != NumElts; ++i)
4331 Idxs[i] = i + (i % 2) * NumElts;
4332
4333 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4334 }
4335
4336 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4337 : IsMask3 ? CI->getArgOperand(2)
4338 : CI->getArgOperand(0);
4339
4340 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4341 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4342 Name.starts_with("avx512.maskz.pternlog.")) {
4343 bool ZeroMask = Name[11] == 'z';
4344 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4345 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4346 Intrinsic::ID IID;
4347 if (VecWidth == 128 && EltWidth == 32)
4348 IID = Intrinsic::x86_avx512_pternlog_d_128;
4349 else if (VecWidth == 256 && EltWidth == 32)
4350 IID = Intrinsic::x86_avx512_pternlog_d_256;
4351 else if (VecWidth == 512 && EltWidth == 32)
4352 IID = Intrinsic::x86_avx512_pternlog_d_512;
4353 else if (VecWidth == 128 && EltWidth == 64)
4354 IID = Intrinsic::x86_avx512_pternlog_q_128;
4355 else if (VecWidth == 256 && EltWidth == 64)
4356 IID = Intrinsic::x86_avx512_pternlog_q_256;
4357 else if (VecWidth == 512 && EltWidth == 64)
4358 IID = Intrinsic::x86_avx512_pternlog_q_512;
4359 else
4360 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4361
4362 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4363 CI->getArgOperand(2), CI->getArgOperand(3)};
4364 Rep = Builder.CreateIntrinsic(IID, Args);
4365 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4366 : CI->getArgOperand(0);
4367 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4368 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4369 Name.starts_with("avx512.maskz.vpmadd52")) {
4370 bool ZeroMask = Name[11] == 'z';
4371 bool High = Name[20] == 'h' || Name[21] == 'h';
4372 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4373 Intrinsic::ID IID;
4374 if (VecWidth == 128 && !High)
4375 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4376 else if (VecWidth == 256 && !High)
4377 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4378 else if (VecWidth == 512 && !High)
4379 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4380 else if (VecWidth == 128 && High)
4381 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4382 else if (VecWidth == 256 && High)
4383 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4384 else if (VecWidth == 512 && High)
4385 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4386 else
4387 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4388
4389 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4390 CI->getArgOperand(2)};
4391 Rep = Builder.CreateIntrinsic(IID, Args);
4392 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4393 : CI->getArgOperand(0);
4394 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4395 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4396 Name.starts_with("avx512.mask.vpermt2var.") ||
4397 Name.starts_with("avx512.maskz.vpermt2var.")) {
4398 bool ZeroMask = Name[11] == 'z';
4399 bool IndexForm = Name[17] == 'i';
4400 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4401 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4402 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4403 Name.starts_with("avx512.mask.vpdpbusds.") ||
4404 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4405 bool ZeroMask = Name[11] == 'z';
4406 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4407 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4408 Intrinsic::ID IID;
4409 if (VecWidth == 128 && !IsSaturating)
4410 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4411 else if (VecWidth == 256 && !IsSaturating)
4412 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4413 else if (VecWidth == 512 && !IsSaturating)
4414 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4415 else if (VecWidth == 128 && IsSaturating)
4416 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4417 else if (VecWidth == 256 && IsSaturating)
4418 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4419 else if (VecWidth == 512 && IsSaturating)
4420 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4421 else
4422 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4423
4424 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4425 CI->getArgOperand(2)};
4426
4427 // Input arguments types were incorrectly set to vectors of i32 before but
4428 // they should be vectors of i8. Insert bit cast when encountering the old
4429 // types
4430 if (Args[1]->getType()->isVectorTy() &&
4431 cast<VectorType>(Args[1]->getType())
4432 ->getElementType()
4433 ->isIntegerTy(32) &&
4434 Args[2]->getType()->isVectorTy() &&
4435 cast<VectorType>(Args[2]->getType())
4436 ->getElementType()
4437 ->isIntegerTy(32)) {
4438 Type *NewArgType = nullptr;
4439 if (VecWidth == 128)
4440 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4441 else if (VecWidth == 256)
4442 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4443 else if (VecWidth == 512)
4444 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4445 else
4446 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4447 CI);
4448
4449 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4450 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4451 }
4452
4453 Rep = Builder.CreateIntrinsic(IID, Args);
4454 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4455 : CI->getArgOperand(0);
4456 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4457 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4458 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4459 Name.starts_with("avx512.mask.vpdpwssds.") ||
4460 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4461 bool ZeroMask = Name[11] == 'z';
4462 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4463 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4464 Intrinsic::ID IID;
4465 if (VecWidth == 128 && !IsSaturating)
4466 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4467 else if (VecWidth == 256 && !IsSaturating)
4468 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4469 else if (VecWidth == 512 && !IsSaturating)
4470 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4471 else if (VecWidth == 128 && IsSaturating)
4472 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4473 else if (VecWidth == 256 && IsSaturating)
4474 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4475 else if (VecWidth == 512 && IsSaturating)
4476 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4477 else
4478 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4479
4480 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4481 CI->getArgOperand(2)};
4482
4483 // Input arguments types were incorrectly set to vectors of i32 before but
4484 // they should be vectors of i16. Insert bit cast when encountering the old
4485 // types
4486 if (Args[1]->getType()->isVectorTy() &&
4487 cast<VectorType>(Args[1]->getType())
4488 ->getElementType()
4489 ->isIntegerTy(32) &&
4490 Args[2]->getType()->isVectorTy() &&
4491 cast<VectorType>(Args[2]->getType())
4492 ->getElementType()
4493 ->isIntegerTy(32)) {
4494 Type *NewArgType = nullptr;
4495 if (VecWidth == 128)
4496 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4497 else if (VecWidth == 256)
4498 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4499 else if (VecWidth == 512)
4500 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4501 else
4502 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4503 CI);
4504
4505 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4506 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4507 }
4508
4509 Rep = Builder.CreateIntrinsic(IID, Args);
4510 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4511 : CI->getArgOperand(0);
4512 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4513 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4514 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4515 Name == "subborrow.u32" || Name == "subborrow.u64") {
4516 Intrinsic::ID IID;
4517 if (Name[0] == 'a' && Name.back() == '2')
4518 IID = Intrinsic::x86_addcarry_32;
4519 else if (Name[0] == 'a' && Name.back() == '4')
4520 IID = Intrinsic::x86_addcarry_64;
4521 else if (Name[0] == 's' && Name.back() == '2')
4522 IID = Intrinsic::x86_subborrow_32;
4523 else if (Name[0] == 's' && Name.back() == '4')
4524 IID = Intrinsic::x86_subborrow_64;
4525 else
4526 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4527
4528 // Make a call with 3 operands.
4529 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4530 CI->getArgOperand(2)};
4531 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4532
4533 // Extract the second result and store it.
4534 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4535 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4536 // Replace the original call result with the first result of the new call.
4537 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4538
4539 CI->replaceAllUsesWith(CF);
4540 Rep = nullptr;
4541 } else if (Name.starts_with("avx512.mask.") &&
4542 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4543 // Rep will be updated by the call in the condition.
4544 } else
4545 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4546
4547 return Rep;
4548}
4549
4550static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4551 Function *F, IRBuilder<> &Builder) {
4552 if (Name.starts_with("neon.bfcvt")) {
4553 if (Name.starts_with("neon.bfcvtn2")) {
4554 SmallVector<int, 32> LoMask(4);
4555 std::iota(LoMask.begin(), LoMask.end(), 0);
4556 SmallVector<int, 32> ConcatMask(8);
4557 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4558 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4559 Value *Trunc =
4560 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4561 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4562 } else if (Name.starts_with("neon.bfcvtn")) {
4563 SmallVector<int, 32> ConcatMask(8);
4564 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4565 Type *V4BF16 =
4566 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4567 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4568 dbgs() << "Trunc: " << *Trunc << "\n";
4569 return Builder.CreateShuffleVector(
4570 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4571 } else {
4572 return Builder.CreateFPTrunc(CI->getOperand(0),
4573 Type::getBFloatTy(F->getContext()));
4574 }
4575 } else if (Name.starts_with("sve.fcvt")) {
4576 Intrinsic::ID NewID =
4577 StringSwitch<Intrinsic::ID>(Name)
4578 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4579 .Case("sve.fcvtnt.bf16f32",
4580 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4581 .Default(Intrinsic::not_intrinsic);
4582 if (NewID == Intrinsic::not_intrinsic)
4583 llvm_unreachable("Unhandled Intrinsic!");
4584
4585 SmallVector<Value *, 3> Args(CI->args());
4586
4587 // The original intrinsics incorrectly used a predicate based on the
4588 // smallest element type rather than the largest.
4589 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4590 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4591
4592 if (Args[1]->getType() != BadPredTy)
4593 llvm_unreachable("Unexpected predicate type!");
4594
4595 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4596 BadPredTy, Args[1]);
4597 Args[1] = Builder.CreateIntrinsic(
4598 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4599
4600 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
4601 CI->getName());
4602 }
4603
4604 llvm_unreachable("Unhandled Intrinsic!");
4605}
4606
4607static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
4608 IRBuilder<> &Builder) {
4609 if (Name == "mve.vctp64.old") {
4610 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
4611 // correct type.
4612 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
4613 CI->getArgOperand(0),
4614 /*FMFSource=*/nullptr, CI->getName());
4615 Value *C1 = Builder.CreateIntrinsic(
4616 Intrinsic::arm_mve_pred_v2i,
4617 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
4618 return Builder.CreateIntrinsic(
4619 Intrinsic::arm_mve_pred_i2v,
4620 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
4621 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4622 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4623 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4624 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4625 Name ==
4626 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4627 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4628 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
4629 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
4630 Name ==
4631 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
4632 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
4633 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
4634 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
4635 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
4636 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
4637 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
4638 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
4639 std::vector<Type *> Tys;
4640 unsigned ID = CI->getIntrinsicID();
4641 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
4642 switch (ID) {
4643 case Intrinsic::arm_mve_mull_int_predicated:
4644 case Intrinsic::arm_mve_vqdmull_predicated:
4645 case Intrinsic::arm_mve_vldr_gather_base_predicated:
4646 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
4647 break;
4648 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4649 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4650 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4651 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
4652 V2I1Ty};
4653 break;
4654 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4655 Tys = {CI->getType(), CI->getOperand(0)->getType(),
4656 CI->getOperand(1)->getType(), V2I1Ty};
4657 break;
4658 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4659 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
4660 CI->getOperand(2)->getType(), V2I1Ty};
4661 break;
4662 case Intrinsic::arm_cde_vcx1q_predicated:
4663 case Intrinsic::arm_cde_vcx1qa_predicated:
4664 case Intrinsic::arm_cde_vcx2q_predicated:
4665 case Intrinsic::arm_cde_vcx2qa_predicated:
4666 case Intrinsic::arm_cde_vcx3q_predicated:
4667 case Intrinsic::arm_cde_vcx3qa_predicated:
4668 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
4669 break;
4670 default:
4671 llvm_unreachable("Unhandled Intrinsic!");
4672 }
4673
4674 std::vector<Value *> Ops;
4675 for (Value *Op : CI->args()) {
4676 Type *Ty = Op->getType();
4677 if (Ty->getScalarSizeInBits() == 1) {
4678 Value *C1 = Builder.CreateIntrinsic(
4679 Intrinsic::arm_mve_pred_v2i,
4680 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
4681 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
4682 }
4683 Ops.push_back(Op);
4684 }
4685
4686 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
4687 CI->getName());
4688 }
4689 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
4690}
4691
4692// These are expected to have the arguments:
4693// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
4694//
4695// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
4696//
4697static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
4698 Function *F, IRBuilder<> &Builder) {
4699 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
4700 // for compatibility.
4701 auto UpgradeLegacyWMMAIUIntrinsicCall =
4702 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
4703 ArrayRef<Type *> OverloadTys) -> Value * {
4704 // Prepare arguments, append clamp=0 for compatibility
4705 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
4706 Args.push_back(Builder.getFalse());
4707
4708 // Insert the declaration for the right overload types
4709 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4710 F->getParent(), F->getIntrinsicID(), OverloadTys);
4711
4712 // Copy operand bundles if any
4713 SmallVector<OperandBundleDef, 1> Bundles;
4714 CI->getOperandBundlesAsDefs(Bundles);
4715
4716 // Create the new call and copy calling properties
4717 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4718 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4719 NewCall->setCallingConv(CI->getCallingConv());
4720 NewCall->setAttributes(CI->getAttributes());
4721 NewCall->setDebugLoc(CI->getDebugLoc());
4722 NewCall->copyMetadata(*CI);
4723 return NewCall;
4724 };
4725
4726 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
4727 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
4728 "intrinsic should have 7 arguments");
4729 Type *T1 = CI->getArgOperand(4)->getType();
4730 Type *T2 = CI->getArgOperand(1)->getType();
4731 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
4732 }
4733 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
4734 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
4735 "intrinsic should have 8 arguments");
4736 Type *T1 = CI->getArgOperand(4)->getType();
4737 Type *T2 = CI->getArgOperand(1)->getType();
4738 Type *T3 = CI->getArgOperand(3)->getType();
4739 Type *T4 = CI->getArgOperand(5)->getType();
4740 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
4741 }
4742
4743 switch (F->getIntrinsicID()) {
4744 default:
4745 break;
4746 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4747 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4748 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4749 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4750 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4751 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
4752 // Drop src0 and src1 modifiers.
4753 const Value *Op0 = CI->getArgOperand(0);
4754 const Value *Op2 = CI->getArgOperand(2);
4755 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
4756 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
4757 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
4758 if (!ModA->isZero() || !ModB->isZero())
4759 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
4760
4761 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
4762 for (int I = 4, E = CI->arg_size(); I < E; ++I)
4763 Args.push_back(CI->getArgOperand(I));
4764
4765 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
4766 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
4767 Overloads.push_back(Args[3]->getType());
4768 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4769 F->getParent(), F->getIntrinsicID(), Overloads);
4770
4771 SmallVector<OperandBundleDef, 1> Bundles;
4772 CI->getOperandBundlesAsDefs(Bundles);
4773
4774 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4775 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4776 NewCall->setCallingConv(CI->getCallingConv());
4777 NewCall->setAttributes(CI->getAttributes());
4778 NewCall->setDebugLoc(CI->getDebugLoc());
4779 NewCall->copyMetadata(*CI);
4780 NewCall->takeName(CI);
4781 return NewCall;
4782 }
4783 }
4784
4785 AtomicRMWInst::BinOp RMWOp =
4786 StringSwitch<AtomicRMWInst::BinOp>(Name)
4787 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
4788 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
4789 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
4790 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
4791 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
4792 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
4793 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
4794 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
4795 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
4796 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
4797 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
4798 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
4799 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
4800
4801 unsigned NumOperands = CI->getNumOperands();
4802 if (NumOperands < 3) // Malformed bitcode.
4803 return nullptr;
4804
4805 Value *Ptr = CI->getArgOperand(0);
4806 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
4807 if (!PtrTy) // Malformed.
4808 return nullptr;
4809
4810 Value *Val = CI->getArgOperand(1);
4811 if (Val->getType() != CI->getType()) // Malformed.
4812 return nullptr;
4813
4814 ConstantInt *OrderArg = nullptr;
4815 bool IsVolatile = false;
4816
4817 // These should have 5 arguments (plus the callee). A separate version of the
4818 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
4819 if (NumOperands > 3)
4820 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
4821
4822 // Ignore scope argument at 3
4823
4824 if (NumOperands > 5) {
4825 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
4826 IsVolatile = !VolatileArg || !VolatileArg->isZero();
4827 }
4828
4829 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
4830 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
4831 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
4832 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
4833 Order = AtomicOrdering::SequentiallyConsistent;
4834
4835 LLVMContext &Ctx = F->getContext();
4836
4837 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
4838 Type *RetTy = CI->getType();
4839 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
4840 if (VT->getElementType()->isIntegerTy(16)) {
4841 VectorType *AsBF16 =
4842 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
4843 Val = Builder.CreateBitCast(Val, AsBF16);
4844 }
4845 }
4846
4847 // The scope argument never really worked correctly. Use agent as the most
4848 // conservative option which should still always produce the instruction.
4849 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
4850 AtomicRMWInst *RMW =
4851 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
4852
4853 unsigned AddrSpace = PtrTy->getAddressSpace();
4854 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
4855 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
4856 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
4857 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
4858 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
4859 }
4860
4861 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
4862 MDBuilder MDB(F->getContext());
4863 MDNode *RangeNotPrivate =
4864 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
4865 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
4866 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
4867 }
4868
4869 if (IsVolatile)
4870 RMW->setVolatile(true);
4871
4872 return Builder.CreateBitCast(RMW, RetTy);
4873}
4874
4875/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
4876/// plain MDNode, as it's the verifier's job to check these are the correct
4877/// types later.
4878static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
4879 if (Op < CI->arg_size()) {
4880 if (MetadataAsValue *MAV =
4881 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
4882 Metadata *MD = MAV->getMetadata();
4883 return dyn_cast_if_present<MDNode>(MD);
4884 }
4885 }
4886 return nullptr;
4887}
4888
4889/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
4890static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
4891 if (Op < CI->arg_size())
4892 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
4893 return MAV->getMetadata();
4894 return nullptr;
4895}
4896
4897/// Convert debug intrinsic calls to non-instruction debug records.
4898/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
4899/// \p CI - The debug intrinsic call.
4900static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
4901 DbgRecord *DR = nullptr;
4902 if (Name == "label") {
4903 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
4904 } else if (Name == "assign") {
4905 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
4906 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
4907 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
4908 unwrapMAVMetadataOp(CI, 4),
4909 /*The address is a Value ref, it will be stored as a Metadata */
4910 unwrapMAVOp(CI, 5));
4911 } else if (Name == "declare") {
4912 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
4913 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
4914 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
4915 } else if (Name == "addr") {
4916 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
4917 MDNode *ExprNode = unwrapMAVOp(CI, 2);
4918 // Don't try to add something to the expression if it's not an expression.
4919 // Instead, allow the verifier to fail later.
4920 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
4921 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
4922 }
4923 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
4924 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
4925 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
4926 } else if (Name == "value") {
4927 // An old version of dbg.value had an extra offset argument.
4928 unsigned VarOp = 1;
4929 unsigned ExprOp = 2;
4930 if (CI->arg_size() == 4) {
4931 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
4932 // Nonzero offset dbg.values get dropped without a replacement.
4933 if (!Offset || !Offset->isNullValue())
4934 return;
4935 VarOp = 2;
4936 ExprOp = 3;
4937 }
4938 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
4939 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
4940 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
4941 nullptr);
4942 }
4943 DR->setDebugLoc(CI->getDebugLoc());
4944 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
4945 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
4946}
4947
4948static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
4949 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
4950 if (!Offset)
4951 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
4952 int64_t OffsetVal = Offset->getSExtValue();
4953 return Builder.CreateIntrinsic(OffsetVal >= 0
4954 ? Intrinsic::vector_splice_left
4955 : Intrinsic::vector_splice_right,
4956 CI->getType(),
4957 {CI->getArgOperand(0), CI->getArgOperand(1),
4958 Builder.getInt32(std::abs(OffsetVal))});
4959}
4960
4961static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
4962 Function *F, IRBuilder<> &Builder) {
4963 if (Name.starts_with("to.fp16")) {
4964 Value *Cast =
4965 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
4966 return Builder.CreateBitCast(Cast, CI->getType());
4967 }
4968
4969 if (Name.starts_with("from.fp16")) {
4970 Value *Cast =
4971 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
4972 return Builder.CreateFPExt(Cast, CI->getType());
4973 }
4974
4975 return nullptr;
4976}
4977
4978/// Upgrade a call to an old intrinsic. All argument and return casting must be
4979/// provided to seamlessly integrate with existing context.
4980void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
4981 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
4982 // checks the callee's function type matches. It's likely we need to handle
4983 // type changes here.
4984 Function *F = dyn_cast<Function>(CI->getCalledOperand());
4985 if (!F)
4986 return;
4987
4988 LLVMContext &C = CI->getContext();
4989 IRBuilder<> Builder(C);
4990 if (isa<FPMathOperator>(CI))
4991 Builder.setFastMathFlags(CI->getFastMathFlags());
4992 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
4993
4994 if (!NewFn) {
4995 // Get the Function's name.
4996 StringRef Name = F->getName();
4997 if (!Name.consume_front("llvm."))
4998 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
4999
5000 bool IsX86 = Name.consume_front("x86.");
5001 bool IsNVVM = Name.consume_front("nvvm.");
5002 bool IsAArch64 = Name.consume_front("aarch64.");
5003 bool IsARM = Name.consume_front("arm.");
5004 bool IsAMDGCN = Name.consume_front("amdgcn.");
5005 bool IsDbg = Name.consume_front("dbg.");
5006 bool IsOldSplice =
5007 (Name.consume_front("experimental.vector.splice") ||
5008 Name.consume_front("vector.splice")) &&
5009 !(Name.starts_with(".left") || Name.starts_with(".right"));
5010 Value *Rep = nullptr;
5011
5012 if (!IsX86 && Name == "stackprotectorcheck") {
5013 Rep = nullptr;
5014 } else if (IsNVVM) {
5015 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5016 } else if (IsX86) {
5017 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5018 } else if (IsAArch64) {
5019 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5020 } else if (IsARM) {
5021 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5022 } else if (IsAMDGCN) {
5023 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5024 } else if (IsDbg) {
5025 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5026 } else if (IsOldSplice) {
5027 Rep = upgradeVectorSplice(CI, Builder);
5028 } else if (Name.consume_front("convert.")) {
5029 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5030 } else {
5031 llvm_unreachable("Unknown function for CallBase upgrade.");
5032 }
5033
5034 if (Rep)
5035 CI->replaceAllUsesWith(Rep);
5036 CI->eraseFromParent();
5037 return;
5038 }
5039
5040 const auto &DefaultCase = [&]() -> void {
5041 if (F == NewFn)
5042 return;
5043
5044 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5045 // Handle generic mangling change.
5046 assert(
5047 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5048 "Unknown function for CallBase upgrade and isn't just a name change");
5049 CI->setCalledFunction(NewFn);
5050 return;
5051 }
5052
5053 // This must be an upgrade from a named to a literal struct.
5054 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5055 assert(OldST != NewFn->getReturnType() &&
5056 "Return type must have changed");
5057 assert(OldST->getNumElements() ==
5058 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5059 "Must have same number of elements");
5060
5061 SmallVector<Value *> Args(CI->args());
5062 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5063 NewCI->setAttributes(CI->getAttributes());
5064 Value *Res = PoisonValue::get(OldST);
5065 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5066 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5067 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5068 }
5069 CI->replaceAllUsesWith(Res);
5070 CI->eraseFromParent();
5071 return;
5072 }
5073
5074 // We're probably about to produce something invalid. Let the verifier catch
5075 // it instead of dying here.
5076 CI->setCalledOperand(
5077 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5078 return;
5079 };
5080 CallInst *NewCall = nullptr;
5081 switch (NewFn->getIntrinsicID()) {
5082 default: {
5083 DefaultCase();
5084 return;
5085 }
5086 case Intrinsic::arm_neon_vst1:
5087 case Intrinsic::arm_neon_vst2:
5088 case Intrinsic::arm_neon_vst3:
5089 case Intrinsic::arm_neon_vst4:
5090 case Intrinsic::arm_neon_vst2lane:
5091 case Intrinsic::arm_neon_vst3lane:
5092 case Intrinsic::arm_neon_vst4lane: {
5093 SmallVector<Value *, 4> Args(CI->args());
5094 NewCall = Builder.CreateCall(NewFn, Args);
5095 break;
5096 }
5097 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5098 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5099 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5100 LLVMContext &Ctx = F->getParent()->getContext();
5101 SmallVector<Value *, 4> Args(CI->args());
5102 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5103 cast<ConstantInt>(Args[3])->getZExtValue());
5104 NewCall = Builder.CreateCall(NewFn, Args);
5105 break;
5106 }
5107 case Intrinsic::aarch64_sve_ld3_sret:
5108 case Intrinsic::aarch64_sve_ld4_sret:
5109 case Intrinsic::aarch64_sve_ld2_sret: {
5110 // Is this a trivial remangle of the name to support ptr address spaces?
5111 if (isa<StructType>(F->getReturnType())) {
5112 DefaultCase();
5113 return;
5114 }
5115
5116 StringRef Name = F->getName();
5117 Name = Name.substr(5);
5118 unsigned N = StringSwitch<unsigned>(Name)
5119 .StartsWith("aarch64.sve.ld2", 2)
5120 .StartsWith("aarch64.sve.ld3", 3)
5121 .StartsWith("aarch64.sve.ld4", 4)
5122 .Default(0);
5123 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5124 unsigned MinElts = RetTy->getMinNumElements() / N;
5125 SmallVector<Value *, 2> Args(CI->args());
5126 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5127 Value *Ret = llvm::PoisonValue::get(RetTy);
5128 for (unsigned I = 0; I < N; I++) {
5129 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5130 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5131 }
5132 NewCall = dyn_cast<CallInst>(Ret);
5133 break;
5134 }
5135
5136 case Intrinsic::coro_end: {
5137 SmallVector<Value *, 3> Args(CI->args());
5138 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5139 NewCall = Builder.CreateCall(NewFn, Args);
5140 break;
5141 }
5142
5143 case Intrinsic::vector_extract: {
5144 StringRef Name = F->getName();
5145 Name = Name.substr(5); // Strip llvm
5146 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5147 DefaultCase();
5148 return;
5149 }
5150 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5151 unsigned MinElts = RetTy->getMinNumElements();
5152 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5153 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5154 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5155 break;
5156 }
5157
5158 case Intrinsic::vector_insert: {
5159 StringRef Name = F->getName();
5160 Name = Name.substr(5);
5161 if (!Name.starts_with("aarch64.sve.tuple")) {
5162 DefaultCase();
5163 return;
5164 }
5165 if (Name.starts_with("aarch64.sve.tuple.set")) {
5166 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5167 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5168 Value *NewIdx =
5169 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5170 NewCall = Builder.CreateCall(
5171 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5172 break;
5173 }
5174 if (Name.starts_with("aarch64.sve.tuple.create")) {
5175 unsigned N = StringSwitch<unsigned>(Name)
5176 .StartsWith("aarch64.sve.tuple.create2", 2)
5177 .StartsWith("aarch64.sve.tuple.create3", 3)
5178 .StartsWith("aarch64.sve.tuple.create4", 4)
5179 .Default(0);
5180 assert(N > 1 && "Create is expected to be between 2-4");
5181 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5182 Value *Ret = llvm::PoisonValue::get(RetTy);
5183 unsigned MinElts = RetTy->getMinNumElements() / N;
5184 for (unsigned I = 0; I < N; I++) {
5185 Value *V = CI->getArgOperand(I);
5186 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5187 }
5188 NewCall = dyn_cast<CallInst>(Ret);
5189 }
5190 break;
5191 }
5192
5193 case Intrinsic::arm_neon_bfdot:
5194 case Intrinsic::arm_neon_bfmmla:
5195 case Intrinsic::arm_neon_bfmlalb:
5196 case Intrinsic::arm_neon_bfmlalt:
5197 case Intrinsic::aarch64_neon_bfdot:
5198 case Intrinsic::aarch64_neon_bfmmla:
5199 case Intrinsic::aarch64_neon_bfmlalb:
5200 case Intrinsic::aarch64_neon_bfmlalt: {
5201 SmallVector<Value *, 3> Args;
5202 assert(CI->arg_size() == 3 &&
5203 "Mismatch between function args and call args");
5204 size_t OperandWidth =
5205 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5206 assert((OperandWidth == 64 || OperandWidth == 128) &&
5207 "Unexpected operand width");
5208 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5209 auto Iter = CI->args().begin();
5210 Args.push_back(*Iter++);
5211 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5212 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5213 NewCall = Builder.CreateCall(NewFn, Args);
5214 break;
5215 }
5216
5217 case Intrinsic::bitreverse:
5218 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5219 break;
5220
5221 case Intrinsic::ctlz:
5222 case Intrinsic::cttz: {
5223 if (CI->arg_size() != 1) {
5224 DefaultCase();
5225 return;
5226 }
5227
5228 NewCall =
5229 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5230 break;
5231 }
5232
5233 case Intrinsic::objectsize: {
5234 Value *NullIsUnknownSize =
5235 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5236 Value *Dynamic =
5237 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5238 NewCall = Builder.CreateCall(
5239 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5240 break;
5241 }
5242
5243 case Intrinsic::ctpop:
5244 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5245 break;
5246 case Intrinsic::dbg_value: {
5247 StringRef Name = F->getName();
5248 Name = Name.substr(5); // Strip llvm.
5249 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5250 if (Name.starts_with("dbg.addr")) {
5251 DIExpression *Expr = cast<DIExpression>(
5252 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5253 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5254 NewCall =
5255 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5256 MetadataAsValue::get(C, Expr)});
5257 break;
5258 }
5259
5260 // Upgrade from the old version that had an extra offset argument.
5261 assert(CI->arg_size() == 4);
5262 // Drop nonzero offsets instead of attempting to upgrade them.
5263 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5264 if (Offset->isNullValue()) {
5265 NewCall = Builder.CreateCall(
5266 NewFn,
5267 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5268 break;
5269 }
5270 CI->eraseFromParent();
5271 return;
5272 }
5273
5274 case Intrinsic::ptr_annotation:
5275 // Upgrade from versions that lacked the annotation attribute argument.
5276 if (CI->arg_size() != 4) {
5277 DefaultCase();
5278 return;
5279 }
5280
5281 // Create a new call with an added null annotation attribute argument.
5282 NewCall = Builder.CreateCall(
5283 NewFn,
5284 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5285 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5286 NewCall->takeName(CI);
5287 CI->replaceAllUsesWith(NewCall);
5288 CI->eraseFromParent();
5289 return;
5290
5291 case Intrinsic::var_annotation:
5292 // Upgrade from versions that lacked the annotation attribute argument.
5293 if (CI->arg_size() != 4) {
5294 DefaultCase();
5295 return;
5296 }
5297 // Create a new call with an added null annotation attribute argument.
5298 NewCall = Builder.CreateCall(
5299 NewFn,
5300 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5301 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5302 NewCall->takeName(CI);
5303 CI->replaceAllUsesWith(NewCall);
5304 CI->eraseFromParent();
5305 return;
5306
5307 case Intrinsic::riscv_aes32dsi:
5308 case Intrinsic::riscv_aes32dsmi:
5309 case Intrinsic::riscv_aes32esi:
5310 case Intrinsic::riscv_aes32esmi:
5311 case Intrinsic::riscv_sm4ks:
5312 case Intrinsic::riscv_sm4ed: {
5313 // The last argument to these intrinsics used to be i8 and changed to i32.
5314 // The type overload for sm4ks and sm4ed was removed.
5315 Value *Arg2 = CI->getArgOperand(2);
5316 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5317 return;
5318
5319 Value *Arg0 = CI->getArgOperand(0);
5320 Value *Arg1 = CI->getArgOperand(1);
5321 if (CI->getType()->isIntegerTy(64)) {
5322 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5323 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5324 }
5325
5326 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5327 cast<ConstantInt>(Arg2)->getZExtValue());
5328
5329 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5330 Value *Res = NewCall;
5331 if (Res->getType() != CI->getType())
5332 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5333 NewCall->takeName(CI);
5334 CI->replaceAllUsesWith(Res);
5335 CI->eraseFromParent();
5336 return;
5337 }
5338 case Intrinsic::nvvm_mapa_shared_cluster: {
5339 // Create a new call with the correct address space.
5340 NewCall =
5341 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5342 Value *Res = NewCall;
5343 Res = Builder.CreateAddrSpaceCast(
5344 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5345 NewCall->takeName(CI);
5346 CI->replaceAllUsesWith(Res);
5347 CI->eraseFromParent();
5348 return;
5349 }
5350 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5351 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5352 // Create a new call with the correct address space.
5353 SmallVector<Value *, 4> Args(CI->args());
5354 Args[0] = Builder.CreateAddrSpaceCast(
5355 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5356
5357 NewCall = Builder.CreateCall(NewFn, Args);
5358 NewCall->takeName(CI);
5359 CI->replaceAllUsesWith(NewCall);
5360 CI->eraseFromParent();
5361 return;
5362 }
5363 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5364 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5365 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5366 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5367 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5368 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5369 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5370 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5371 SmallVector<Value *, 16> Args(CI->args());
5372
5373 // Create AddrSpaceCast to shared_cluster if needed.
5374 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5375 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5376 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5377 Args[0] = Builder.CreateAddrSpaceCast(
5378 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5379
5380 // Attach the flag argument for cta_group, with a
5381 // default value of 0. This handles case (2) in
5382 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5383 size_t NumArgs = CI->arg_size();
5384 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5385 if (!FlagArg->getType()->isIntegerTy(1))
5386 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5387
5388 NewCall = Builder.CreateCall(NewFn, Args);
5389 NewCall->takeName(CI);
5390 CI->replaceAllUsesWith(NewCall);
5391 CI->eraseFromParent();
5392 return;
5393 }
5394 case Intrinsic::riscv_sha256sig0:
5395 case Intrinsic::riscv_sha256sig1:
5396 case Intrinsic::riscv_sha256sum0:
5397 case Intrinsic::riscv_sha256sum1:
5398 case Intrinsic::riscv_sm3p0:
5399 case Intrinsic::riscv_sm3p1: {
5400 // The last argument to these intrinsics used to be i8 and changed to i32.
5401 // The type overload for sm4ks and sm4ed was removed.
5402 if (!CI->getType()->isIntegerTy(64))
5403 return;
5404
5405 Value *Arg =
5406 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
5407
5408 NewCall = Builder.CreateCall(NewFn, Arg);
5409 Value *Res =
5410 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5411 NewCall->takeName(CI);
5412 CI->replaceAllUsesWith(Res);
5413 CI->eraseFromParent();
5414 return;
5415 }
5416
5417 case Intrinsic::x86_xop_vfrcz_ss:
5418 case Intrinsic::x86_xop_vfrcz_sd:
5419 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
5420 break;
5421
5422 case Intrinsic::x86_xop_vpermil2pd:
5423 case Intrinsic::x86_xop_vpermil2ps:
5424 case Intrinsic::x86_xop_vpermil2pd_256:
5425 case Intrinsic::x86_xop_vpermil2ps_256: {
5426 SmallVector<Value *, 4> Args(CI->args());
5427 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
5428 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
5429 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
5430 NewCall = Builder.CreateCall(NewFn, Args);
5431 break;
5432 }
5433
5434 case Intrinsic::x86_sse41_ptestc:
5435 case Intrinsic::x86_sse41_ptestz:
5436 case Intrinsic::x86_sse41_ptestnzc: {
5437 // The arguments for these intrinsics used to be v4f32, and changed
5438 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
5439 // So, the only thing required is a bitcast for both arguments.
5440 // First, check the arguments have the old type.
5441 Value *Arg0 = CI->getArgOperand(0);
5442 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
5443 return;
5444
5445 // Old intrinsic, add bitcasts
5446 Value *Arg1 = CI->getArgOperand(1);
5447
5448 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
5449
5450 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
5451 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
5452
5453 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
5454 break;
5455 }
5456
5457 case Intrinsic::x86_rdtscp: {
5458 // This used to take 1 arguments. If we have no arguments, it is already
5459 // upgraded.
5460 if (CI->getNumOperands() == 0)
5461 return;
5462
5463 NewCall = Builder.CreateCall(NewFn);
5464 // Extract the second result and store it.
5465 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5466 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
5467 // Replace the original call result with the first result of the new call.
5468 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
5469
5470 NewCall->takeName(CI);
5471 CI->replaceAllUsesWith(TSC);
5472 CI->eraseFromParent();
5473 return;
5474 }
5475
5476 case Intrinsic::x86_sse41_insertps:
5477 case Intrinsic::x86_sse41_dppd:
5478 case Intrinsic::x86_sse41_dpps:
5479 case Intrinsic::x86_sse41_mpsadbw:
5480 case Intrinsic::x86_avx_dp_ps_256:
5481 case Intrinsic::x86_avx2_mpsadbw: {
5482 // Need to truncate the last argument from i32 to i8 -- this argument models
5483 // an inherently 8-bit immediate operand to these x86 instructions.
5484 SmallVector<Value *, 4> Args(CI->args());
5485
5486 // Replace the last argument with a trunc.
5487 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
5488 NewCall = Builder.CreateCall(NewFn, Args);
5489 break;
5490 }
5491
5492 case Intrinsic::x86_avx512_mask_cmp_pd_128:
5493 case Intrinsic::x86_avx512_mask_cmp_pd_256:
5494 case Intrinsic::x86_avx512_mask_cmp_pd_512:
5495 case Intrinsic::x86_avx512_mask_cmp_ps_128:
5496 case Intrinsic::x86_avx512_mask_cmp_ps_256:
5497 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
5498 SmallVector<Value *, 4> Args(CI->args());
5499 unsigned NumElts =
5500 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
5501 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
5502
5503 NewCall = Builder.CreateCall(NewFn, Args);
5504 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
5505
5506 NewCall->takeName(CI);
5507 CI->replaceAllUsesWith(Res);
5508 CI->eraseFromParent();
5509 return;
5510 }
5511
5512 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
5513 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
5514 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
5515 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
5516 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
5517 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
5518 SmallVector<Value *, 4> Args(CI->args());
5519 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
5520 if (NewFn->getIntrinsicID() ==
5521 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
5522 Args[1] = Builder.CreateBitCast(
5523 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5524
5525 NewCall = Builder.CreateCall(NewFn, Args);
5526 Value *Res = Builder.CreateBitCast(
5527 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
5528
5529 NewCall->takeName(CI);
5530 CI->replaceAllUsesWith(Res);
5531 CI->eraseFromParent();
5532 return;
5533 }
5534 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
5535 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
5536 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
5537 SmallVector<Value *, 4> Args(CI->args());
5538 unsigned NumElts =
5539 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
5540 Args[1] = Builder.CreateBitCast(
5541 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5542 Args[2] = Builder.CreateBitCast(
5543 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5544
5545 NewCall = Builder.CreateCall(NewFn, Args);
5546 break;
5547 }
5548
5549 case Intrinsic::thread_pointer: {
5550 NewCall = Builder.CreateCall(NewFn, {});
5551 break;
5552 }
5553
5554 case Intrinsic::memcpy:
5555 case Intrinsic::memmove:
5556 case Intrinsic::memset: {
5557 // We have to make sure that the call signature is what we're expecting.
5558 // We only want to change the old signatures by removing the alignment arg:
5559 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
5560 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
5561 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
5562 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
5563 // Note: i8*'s in the above can be any pointer type
5564 if (CI->arg_size() != 5) {
5565 DefaultCase();
5566 return;
5567 }
5568 // Remove alignment argument (3), and add alignment attributes to the
5569 // dest/src pointers.
5570 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
5571 CI->getArgOperand(2), CI->getArgOperand(4)};
5572 NewCall = Builder.CreateCall(NewFn, Args);
5573 AttributeList OldAttrs = CI->getAttributes();
5574 AttributeList NewAttrs = AttributeList::get(
5575 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
5576 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
5577 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
5578 NewCall->setAttributes(NewAttrs);
5579 auto *MemCI = cast<MemIntrinsic>(NewCall);
5580 // All mem intrinsics support dest alignment.
5581 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
5582 MemCI->setDestAlignment(Align->getMaybeAlignValue());
5583 // Memcpy/Memmove also support source alignment.
5584 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
5585 MTI->setSourceAlignment(Align->getMaybeAlignValue());
5586 break;
5587 }
5588
5589 case Intrinsic::masked_load:
5590 case Intrinsic::masked_gather:
5591 case Intrinsic::masked_store:
5592 case Intrinsic::masked_scatter: {
5593 if (CI->arg_size() != 4) {
5594 DefaultCase();
5595 return;
5596 }
5597
5598 auto GetMaybeAlign = [](Value *Op) {
5599 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
5600 uint64_t Val = CI->getZExtValue();
5601 if (Val == 0)
5602 return MaybeAlign();
5603 if (isPowerOf2_64(Val))
5604 return MaybeAlign(Val);
5605 }
5606 reportFatalUsageError("Invalid alignment argument");
5607 };
5608 auto GetAlign = [&](Value *Op) {
5609 MaybeAlign Align = GetMaybeAlign(Op);
5610 if (Align)
5611 return *Align;
5612 reportFatalUsageError("Invalid zero alignment argument");
5613 };
5614
5615 const DataLayout &DL = CI->getDataLayout();
5616 switch (NewFn->getIntrinsicID()) {
5617 case Intrinsic::masked_load:
5618 NewCall = Builder.CreateMaskedLoad(
5619 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
5620 CI->getArgOperand(2), CI->getArgOperand(3));
5621 break;
5622 case Intrinsic::masked_gather:
5623 NewCall = Builder.CreateMaskedGather(
5624 CI->getType(), CI->getArgOperand(0),
5625 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
5626 CI->getType()->getScalarType()),
5627 CI->getArgOperand(2), CI->getArgOperand(3));
5628 break;
5629 case Intrinsic::masked_store:
5630 NewCall = Builder.CreateMaskedStore(
5631 CI->getArgOperand(0), CI->getArgOperand(1),
5632 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
5633 break;
5634 case Intrinsic::masked_scatter:
5635 NewCall = Builder.CreateMaskedScatter(
5636 CI->getArgOperand(0), CI->getArgOperand(1),
5637 DL.getValueOrABITypeAlignment(
5638 GetMaybeAlign(CI->getArgOperand(2)),
5639 CI->getArgOperand(0)->getType()->getScalarType()),
5640 CI->getArgOperand(3));
5641 break;
5642 default:
5643 llvm_unreachable("Unexpected intrinsic ID");
5644 }
5645 // Previous metadata is still valid.
5646 NewCall->copyMetadata(*CI);
5647 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5648 break;
5649 }
5650
5651 case Intrinsic::lifetime_start:
5652 case Intrinsic::lifetime_end: {
5653 if (CI->arg_size() != 2) {
5654 DefaultCase();
5655 return;
5656 }
5657
5658 Value *Ptr = CI->getArgOperand(1);
5659 // Try to strip pointer casts, such that the lifetime works on an alloca.
5660 Ptr = Ptr->stripPointerCasts();
5661 if (isa<AllocaInst>(Ptr)) {
5662 // Don't use NewFn, as we might have looked through an addrspacecast.
5663 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
5664 NewCall = Builder.CreateLifetimeStart(Ptr);
5665 else
5666 NewCall = Builder.CreateLifetimeEnd(Ptr);
5667 break;
5668 }
5669
5670 // Otherwise remove the lifetime marker.
5671 CI->eraseFromParent();
5672 return;
5673 }
5674
5675 case Intrinsic::x86_avx512_vpdpbusd_128:
5676 case Intrinsic::x86_avx512_vpdpbusd_256:
5677 case Intrinsic::x86_avx512_vpdpbusd_512:
5678 case Intrinsic::x86_avx512_vpdpbusds_128:
5679 case Intrinsic::x86_avx512_vpdpbusds_256:
5680 case Intrinsic::x86_avx512_vpdpbusds_512:
5681 case Intrinsic::x86_avx2_vpdpbssd_128:
5682 case Intrinsic::x86_avx2_vpdpbssd_256:
5683 case Intrinsic::x86_avx10_vpdpbssd_512:
5684 case Intrinsic::x86_avx2_vpdpbssds_128:
5685 case Intrinsic::x86_avx2_vpdpbssds_256:
5686 case Intrinsic::x86_avx10_vpdpbssds_512:
5687 case Intrinsic::x86_avx2_vpdpbsud_128:
5688 case Intrinsic::x86_avx2_vpdpbsud_256:
5689 case Intrinsic::x86_avx10_vpdpbsud_512:
5690 case Intrinsic::x86_avx2_vpdpbsuds_128:
5691 case Intrinsic::x86_avx2_vpdpbsuds_256:
5692 case Intrinsic::x86_avx10_vpdpbsuds_512:
5693 case Intrinsic::x86_avx2_vpdpbuud_128:
5694 case Intrinsic::x86_avx2_vpdpbuud_256:
5695 case Intrinsic::x86_avx10_vpdpbuud_512:
5696 case Intrinsic::x86_avx2_vpdpbuuds_128:
5697 case Intrinsic::x86_avx2_vpdpbuuds_256:
5698 case Intrinsic::x86_avx10_vpdpbuuds_512: {
5699 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
5700 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5701 CI->getArgOperand(2)};
5702 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
5703 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5704 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5705
5706 NewCall = Builder.CreateCall(NewFn, Args);
5707 break;
5708 }
5709 case Intrinsic::x86_avx512_vpdpwssd_128:
5710 case Intrinsic::x86_avx512_vpdpwssd_256:
5711 case Intrinsic::x86_avx512_vpdpwssd_512:
5712 case Intrinsic::x86_avx512_vpdpwssds_128:
5713 case Intrinsic::x86_avx512_vpdpwssds_256:
5714 case Intrinsic::x86_avx512_vpdpwssds_512:
5715 case Intrinsic::x86_avx2_vpdpwsud_128:
5716 case Intrinsic::x86_avx2_vpdpwsud_256:
5717 case Intrinsic::x86_avx10_vpdpwsud_512:
5718 case Intrinsic::x86_avx2_vpdpwsuds_128:
5719 case Intrinsic::x86_avx2_vpdpwsuds_256:
5720 case Intrinsic::x86_avx10_vpdpwsuds_512:
5721 case Intrinsic::x86_avx2_vpdpwusd_128:
5722 case Intrinsic::x86_avx2_vpdpwusd_256:
5723 case Intrinsic::x86_avx10_vpdpwusd_512:
5724 case Intrinsic::x86_avx2_vpdpwusds_128:
5725 case Intrinsic::x86_avx2_vpdpwusds_256:
5726 case Intrinsic::x86_avx10_vpdpwusds_512:
5727 case Intrinsic::x86_avx2_vpdpwuud_128:
5728 case Intrinsic::x86_avx2_vpdpwuud_256:
5729 case Intrinsic::x86_avx10_vpdpwuud_512:
5730 case Intrinsic::x86_avx2_vpdpwuuds_128:
5731 case Intrinsic::x86_avx2_vpdpwuuds_256:
5732 case Intrinsic::x86_avx10_vpdpwuuds_512:
5733 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
5734 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5735 CI->getArgOperand(2)};
5736 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
5737 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5738 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5739
5740 NewCall = Builder.CreateCall(NewFn, Args);
5741 break;
5742 }
5743 assert(NewCall && "Should have either set this variable or returned through "
5744 "the default case");
5745 NewCall->takeName(CI);
5746 CI->replaceAllUsesWith(NewCall);
5747 CI->eraseFromParent();
5748}
5749
5750void llvm::UpgradeCallsToIntrinsic(Function *F) {
5751 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
5752
5753 // Check if this function should be upgraded and get the replacement function
5754 // if there is one.
5755 Function *NewFn;
5756 if (UpgradeIntrinsicFunction(F, NewFn)) {
5757 // Replace all users of the old function with the new function or new
5758 // instructions. This is not a range loop because the call is deleted.
5759 for (User *U : make_early_inc_range(F->users()))
5760 if (CallBase *CB = dyn_cast<CallBase>(U))
5761 UpgradeIntrinsicCall(CB, NewFn);
5762
5763 // Remove old function, no longer used, from the module.
5764 if (F != NewFn)
5765 F->eraseFromParent();
5766 }
5767}
5768
5769MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
5770 const unsigned NumOperands = MD.getNumOperands();
5771 if (NumOperands == 0)
5772 return &MD; // Invalid, punt to a verifier error.
5773
5774 // Check if the tag uses struct-path aware TBAA format.
5775 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
5776 return &MD;
5777
5778 auto &Context = MD.getContext();
5779 if (NumOperands == 3) {
5780 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
5781 MDNode *ScalarType = MDNode::get(Context, Elts);
5782 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
5783 Metadata *Elts2[] = {ScalarType, ScalarType,
5784 ConstantAsMetadata::get(
5785 Constant::getNullValue(Type::getInt64Ty(Context))),
5786 MD.getOperand(2)};
5787 return MDNode::get(Context, Elts2);
5788 }
5789 // Create a MDNode <MD, MD, offset 0>
5790 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
5791 Type::getInt64Ty(Context)))};
5792 return MDNode::get(Context, Elts);
5793}
5794
5795Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
5796 Instruction *&Temp) {
5797 if (Opc != Instruction::BitCast)
5798 return nullptr;
5799
5800 Temp = nullptr;
5801 Type *SrcTy = V->getType();
5802 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
5803 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
5804 LLVMContext &Context = V->getContext();
5805
5806 // We have no information about target data layout, so we assume that
5807 // the maximum pointer size is 64bit.
5808 Type *MidTy = Type::getInt64Ty(Context);
5809 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
5810
5811 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
5812 }
5813
5814 return nullptr;
5815}
5816
5817Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
5818 if (Opc != Instruction::BitCast)
5819 return nullptr;
5820
5821 Type *SrcTy = C->getType();
5822 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
5823 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
5824 LLVMContext &Context = C->getContext();
5825
5826 // We have no information about target data layout, so we assume that
5827 // the maximum pointer size is 64bit.
5828 Type *MidTy = Type::getInt64Ty(Context);
5829
5830 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
5831 DestTy);
5832 }
5833
5834 return nullptr;
5835}
5836
5837/// Check the debug info version number, if it is out-dated, drop the debug
5838/// info. Return true if module is modified.
5839bool llvm::UpgradeDebugInfo(Module &M) {
5840 if (DisableAutoUpgradeDebugInfo)
5841 return false;
5842
5843 llvm::TimeTraceScope timeScope("Upgrade debug info");
5844 // We need to get metadata before the module is verified (i.e., getModuleFlag
5845 // makes assumptions that we haven't verified yet). Carefully extract the flag
5846 // from the metadata.
5847 unsigned Version = 0;
5848 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
5849 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
5850 if (Flag->getNumOperands() < 3)
5851 return false;
5852 if (MDString *K = dyn_cast_or_null<MDString>(Flag->getOperand(1)))
5853 return K->getString() == "Debug Info Version";
5854 return false;
5855 });
5856 if (OpIt != ModFlags->op_end()) {
5857 const MDOperand &ValOp = (*OpIt)->getOperand(2);
5858 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
5859 Version = CI->getZExtValue();
5860 }
5861 }
5862
5863 if (Version == DEBUG_METADATA_VERSION) {
5864 bool BrokenDebugInfo = false;
5865 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
5866 report_fatal_error("Broken module found, compilation aborted!");
5867 if (!BrokenDebugInfo)
5868 // Everything is ok.
5869 return false;
5870 else {
5871 // Diagnose malformed debug info.
5872 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
5873 M.getContext().diagnose(Diag);
5874 }
5875 }
5876 bool Modified = StripDebugInfo(M);
5877 if (Modified && Version != DEBUG_METADATA_VERSION) {
5878 // Diagnose a version mismatch.
5879 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
5880 M.getContext().diagnose(DiagVersion);
5881 }
5882 return Modified;
5883}
5884
5885static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
5886 GlobalValue *GV, const Metadata *V) {
5887 Function *F = cast<Function>(GV);
5888
5889 constexpr StringLiteral DefaultValue = "1";
5890 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
5891 unsigned Length = 0;
5892
5893 if (F->hasFnAttribute(Attr)) {
5894 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
5895 // parse these elements placing them into Vect3
5896 StringRef S = F->getFnAttribute(Attr).getValueAsString();
5897 for (; Length < 3 && !S.empty(); Length++) {
5898 auto [Part, Rest] = S.split(',');
5899 Vect3[Length] = Part.trim();
5900 S = Rest;
5901 }
5902 }
5903
5904 const unsigned Dim = DimC - 'x';
5905 assert(Dim < 3 && "Unexpected dim char");
5906
5907 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
5908
5909 // local variable required for StringRef in Vect3 to point to.
5910 const std::string VStr = llvm::utostr(VInt);
5911 Vect3[Dim] = VStr;
5912 Length = std::max(Length, Dim + 1);
5913
5914 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
5915 F->addFnAttr(Attr, NewAttr);
5916}
5917
5918static inline bool isXYZ(StringRef S) {
5919 return S == "x" || S == "y" || S == "z";
5920}
5921
5922bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
5923 const Metadata *V) {
5924 if (K == "kernel") {
5925 if (!mdconst::extract<ConstantInt>(V)->isZero())
5926 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
5927 return true;
5928 }
5929 if (K == "align") {
5930 // V is a bitfeild specifying two 16-bit values. The alignment value is
5931 // specfied in low 16-bits, The index is specified in the high bits. For the
5932 // index, 0 indicates the return value while higher values correspond to
5933 // each parameter (idx = param + 1).
5934 const uint64_t AlignIdxValuePair =
5935 mdconst::extract<ConstantInt>(V)->getZExtValue();
5936 const unsigned Idx = (AlignIdxValuePair >> 16);
5937 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
5938 cast<Function>(GV)->addAttributeAtIndex(
5939 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
5940 return true;
5941 }
5942 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
5943 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
5944 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
5945 return true;
5946 }
5947 if (K == "minctasm") {
5948 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
5949 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
5950 return true;
5951 }
5952 if (K == "maxnreg") {
5953 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
5954 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
5955 return true;
5956 }
5957 if (K.consume_front("maxntid") && isXYZ(K)) {
5958 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
5959 return true;
5960 }
5961 if (K.consume_front("reqntid") && isXYZ(K)) {
5962 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
5963 return true;
5964 }
5965 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
5966 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
5967 return true;
5968 }
5969 if (K == "grid_constant") {
5970 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
5971 for (const auto &Op : cast<MDNode>(V)->operands()) {
5972 // For some reason, the index is 1-based in the metadata. Good thing we're
5973 // able to auto-upgrade it!
5974 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
5975 cast<Function>(GV)->addParamAttr(Index, Attr);
5976 }
5977 return true;
5978 }
5979
5980 return false;
5981}
5982
5983void llvm::UpgradeNVVMAnnotations(Module &M) {
5984 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
5985 if (!NamedMD)
5986 return;
5987
5988 SmallVector<MDNode *, 8> NewNodes;
5989 SmallPtrSet<const MDNode *, 8> SeenNodes;
5990 for (MDNode *MD : NamedMD->operands()) {
5991 if (!SeenNodes.insert(MD).second)
5992 continue;
5993
5994 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
5995 if (!GV)
5996 continue;
5997
5998 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
5999
6000 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6001 // Each nvvm.annotations metadata entry will be of the following form:
6002 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6003 // start index = 1, to skip the global variable key
6004 // increment = 2, to skip the value for each property-value pairs
6005 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6006 MDString *K = cast<MDString>(MD->getOperand(j));
6007 const MDOperand &V = MD->getOperand(j + 1);
6008 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6009 if (!Upgraded)
6010 NewOperands.append({K, V});
6011 }
6012
6013 if (NewOperands.size() > 1)
6014 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6015 }
6016
6017 NamedMD->clearOperands();
6018 for (MDNode *N : NewNodes)
6019 NamedMD->addOperand(N);
6020}
6021
6022/// This checks for objc retain release marker which should be upgraded. It
6023/// returns true if module is modified.
6024static bool upgradeRetainReleaseMarker(Module &M) {
6025 bool Changed = false;
6026 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6027 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6028 if (ModRetainReleaseMarker) {
6029 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6030 if (Op) {
6031 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6032 if (ID) {
6033 SmallVector<StringRef, 4> ValueComp;
6034 ID->getString().split(ValueComp, "#");
6035 if (ValueComp.size() == 2) {
6036 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6037 ID = MDString::get(M.getContext(), NewValue);
6038 }
6039 M.addModuleFlag(Module::Error, MarkerKey, ID);
6040 M.eraseNamedMetadata(ModRetainReleaseMarker);
6041 Changed = true;
6042 }
6043 }
6044 }
6045 return Changed;
6046}
6047
6048void llvm::UpgradeARCRuntime(Module &M) {
6049 // This lambda converts normal function calls to ARC runtime functions to
6050 // intrinsic calls.
6051 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6052 llvm::Intrinsic::ID IntrinsicFunc) {
6053 Function *Fn = M.getFunction(OldFunc);
6054
6055 if (!Fn)
6056 return;
6057
6058 Function *NewFn =
6059 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6060
6061 for (User *U : make_early_inc_range(Fn->users())) {
6062 CallInst *CI = dyn_cast<CallInst>(U);
6063 if (!CI || CI->getCalledFunction() != Fn)
6064 continue;
6065
6066 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6067 FunctionType *NewFuncTy = NewFn->getFunctionType();
6068 SmallVector<Value *, 2> Args;
6069
6070 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6071 // value to the return type of the old function.
6072 if (NewFuncTy->getReturnType() != CI->getType() &&
6073 !CastInst::castIsValid(Instruction::BitCast, CI,
6074 NewFuncTy->getReturnType()))
6075 continue;
6076
6077 bool InvalidCast = false;
6078
6079 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6080 Value *Arg = CI->getArgOperand(I);
6081
6082 // Bitcast argument to the parameter type of the new function if it's
6083 // not a variadic argument.
6084 if (I < NewFuncTy->getNumParams()) {
6085 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6086 // to the parameter type of the new function.
6087 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6088 NewFuncTy->getParamType(I))) {
6089 InvalidCast = true;
6090 break;
6091 }
6092 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6093 }
6094 Args.push_back(Arg);
6095 }
6096
6097 if (InvalidCast)
6098 continue;
6099
6100 // Create a call instruction that calls the new function.
6101 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6102 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6103 NewCall->takeName(CI);
6104
6105 // Bitcast the return value back to the type of the old call.
6106 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6107
6108 if (!CI->use_empty())
6109 CI->replaceAllUsesWith(NewRetVal);
6110 CI->eraseFromParent();
6111 }
6112
6113 if (Fn->use_empty())
6114 Fn->eraseFromParent();
6115 };
6116
6117 // Unconditionally convert a call to "clang.arc.use" to a call to
6118 // "llvm.objc.clang.arc.use".
6119 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6120
6121 // Upgrade the retain release marker. If there is no need to upgrade
6122 // the marker, that means either the module is already new enough to contain
6123 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6124 if (!upgradeRetainReleaseMarker(M))
6125 return;
6126
6127 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6128 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6129 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6130 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6131 {"objc_autoreleaseReturnValue",
6132 llvm::Intrinsic::objc_autoreleaseReturnValue},
6133 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6134 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6135 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6136 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6137 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6138 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6139 {"objc_release", llvm::Intrinsic::objc_release},
6140 {"objc_retain", llvm::Intrinsic::objc_retain},
6141 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6142 {"objc_retainAutoreleaseReturnValue",
6143 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6144 {"objc_retainAutoreleasedReturnValue",
6145 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6146 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6147 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6148 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6149 {"objc_unsafeClaimAutoreleasedReturnValue",
6150 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6151 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6152 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6153 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6154 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6155 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6156 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6157 {"objc_arc_annotation_topdown_bbstart",
6158 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6159 {"objc_arc_annotation_topdown_bbend",
6160 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6161 {"objc_arc_annotation_bottomup_bbstart",
6162 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6163 {"objc_arc_annotation_bottomup_bbend",
6164 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6165
6166 for (auto &I : RuntimeFuncs)
6167 UpgradeToIntrinsic(I.first, I.second);
6168}
6169
6170bool llvm::UpgradeModuleFlags(Module &M) {
6171 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6172 if (!ModFlags)
6173 return false;
6174
6175 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
6176 bool HasSwiftVersionFlag = false;
6177 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6178 uint32_t SwiftABIVersion;
6179 auto Int8Ty = Type::getInt8Ty(M.getContext());
6180 auto Int32Ty = Type::getInt32Ty(M.getContext());
6181
6182 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6183 MDNode *Op = ModFlags->getOperand(I);
6184 if (Op->getNumOperands() != 3)
6185 continue;
6186 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6187 if (!ID)
6188 continue;
6189 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6190 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6191 Type::getInt32Ty(M.getContext()), B)),
6192 MDString::get(M.getContext(), ID->getString()),
6193 Op->getOperand(2)};
6194 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6195 Changed = true;
6196 };
6197
6198 if (ID->getString() == "Objective-C Image Info Version")
6199 HasObjCFlag = true;
6200 if (ID->getString() == "Objective-C Class Properties")
6201 HasClassProperties = true;
6202 // Upgrade PIC from Error/Max to Min.
6203 if (ID->getString() == "PIC Level") {
6204 if (auto *Behavior =
6205 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6206 uint64_t V = Behavior->getLimitedValue();
6207 if (V == Module::Error || V == Module::Max)
6208 SetBehavior(Module::Min);
6209 }
6210 }
6211 // Upgrade "PIE Level" from Error to Max.
6212 if (ID->getString() == "PIE Level")
6213 if (auto *Behavior =
6214 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
6215 if (Behavior->getLimitedValue() == Module::Error)
6216 SetBehavior(Module::Max);
6217
6218 // Upgrade branch protection and return address signing module flags. The
6219 // module flag behavior for these fields were Error and now they are Min.
6220 if (ID->getString() == "branch-target-enforcement" ||
6221 ID->getString().starts_with("sign-return-address")) {
6222 if (auto *Behavior =
6223 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6224 if (Behavior->getLimitedValue() == Module::Error) {
6225 Type *Int32Ty = Type::getInt32Ty(M.getContext());
6226 Metadata *Ops[3] = {
6227 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
6228 Op->getOperand(1), Op->getOperand(2)};
6229 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6230 Changed = true;
6231 }
6232 }
6233 }
6234
6235 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
6236 // section name so that llvm-lto will not complain about mismatching
6237 // module flags that is functionally the same.
6238 if (ID->getString() == "Objective-C Image Info Section") {
6239 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
6240 SmallVector<StringRef, 4> ValueComp;
6241 Value->getString().split(ValueComp, " ");
6242 if (ValueComp.size() != 1) {
6243 std::string NewValue;
6244 for (auto &S : ValueComp)
6245 NewValue += S.str();
6246 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
6247 MDString::get(M.getContext(), NewValue)};
6248 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6249 Changed = true;
6250 }
6251 }
6252 }
6253
6254 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
6255 // If the higher bits are set, it adds new module flag for swift info.
6256 if (ID->getString() == "Objective-C Garbage Collection") {
6257 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
6258 if (Md) {
6259 assert(Md->getValue() && "Expected non-empty metadata");
6260 auto Type = Md->getValue()->getType();
6261 if (Type == Int8Ty)
6262 continue;
6263 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
6264 if ((Val & 0xff) != Val) {
6265 HasSwiftVersionFlag = true;
6266 SwiftABIVersion = (Val & 0xff00) >> 8;
6267 SwiftMajorVersion = (Val & 0xff000000) >> 24;
6268 SwiftMinorVersion = (Val & 0xff0000) >> 16;
6269 }
6270 Metadata *Ops[3] = {
6271 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
6272 Op->getOperand(1),
6273 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
6274 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6275 Changed = true;
6276 }
6277 }
6278
6279 if (ID->getString() == "amdgpu_code_object_version") {
6280 Metadata *Ops[3] = {
6281 Op->getOperand(0),
6282 MDString::get(M.getContext(), "amdhsa_code_object_version"),
6283 Op->getOperand(2)};
6284 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6285 Changed = true;
6286 }
6287 }
6288
6289 // "Objective-C Class Properties" is recently added for Objective-C. We
6290 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
6291 // flag of value 0, so we can correclty downgrade this flag when trying to
6292 // link an ObjC bitcode without this module flag with an ObjC bitcode with
6293 // this module flag.
6294 if (HasObjCFlag && !HasClassProperties) {
6295 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
6296 (uint32_t)0);
6297 Changed = true;
6298 }
6299
6300 if (HasSwiftVersionFlag) {
6301 M.addModuleFlag(Module::Error, "Swift ABI Version",
6302 SwiftABIVersion);
6303 M.addModuleFlag(Module::Error, "Swift Major Version",
6304 ConstantInt::get(Int8Ty, SwiftMajorVersion));
6305 M.addModuleFlag(Module::Error, "Swift Minor Version",
6306 ConstantInt::get(Int8Ty, SwiftMinorVersion));
6307 Changed = true;
6308 }
6309
6310 return Changed;
6311}
6312
6313void llvm::UpgradeSectionAttributes(Module &M) {
6314 auto TrimSpaces = [](StringRef Section) -> std::string {
6315 SmallVector<StringRef, 5> Components;
6316 Section.split(Components, ',');
6317
6318 SmallString<32> Buffer;
6319 raw_svector_ostream OS(Buffer);
6320
6321 for (auto Component : Components)
6322 OS << ',' << Component.trim();
6323
6324 return std::string(OS.str().substr(1));
6325 };
6326
6327 for (auto &GV : M.globals()) {
6328 if (!GV.hasSection())
6329 continue;
6330
6331 StringRef Section = GV.getSection();
6332
6333 if (!Section.starts_with("__DATA, __objc_catlist"))
6334 continue;
6335
6336 // __DATA, __objc_catlist, regular, no_dead_strip
6337 // __DATA,__objc_catlist,regular,no_dead_strip
6338 GV.setSection(TrimSpaces(Section));
6339 }
6340}
6341
6342namespace {
6343// Prior to LLVM 10.0, the strictfp attribute could be used on individual
6344// callsites within a function that did not also have the strictfp attribute.
6345// Since 10.0, if strict FP semantics are needed within a function, the
6346// function must have the strictfp attribute and all calls within the function
6347// must also have the strictfp attribute. This latter restriction is
6348// necessary to prevent unwanted libcall simplification when a function is
6349// being cloned (such as for inlining).
6350//
6351// The "dangling" strictfp attribute usage was only used to prevent constant
6352// folding and other libcall simplification. The nobuiltin attribute on the
6353// callsite has the same effect.
6354struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
6355 StrictFPUpgradeVisitor() = default;
6356
6357 void visitCallBase(CallBase &Call) {
6358 if (!Call.isStrictFP())
6359 return;
6360 if (isa<ConstrainedFPIntrinsic>(&Call))
6361 return;
6362 // If we get here, the caller doesn't have the strictfp attribute
6363 // but this callsite does. Replace the strictfp attribute with nobuiltin.
6364 Call.removeFnAttr(Attribute::StrictFP);
6365 Call.addFnAttr(Attribute::NoBuiltin);
6366 }
6367};
6368
6369/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
6370struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
6371 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
6372 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
6373
6374 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
6375 if (!RMW.isFloatingPointOperation())
6376 return;
6377
6378 MDNode *Empty = MDNode::get(RMW.getContext(), {});
6379 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
6380 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
6381 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
6382 }
6383};
6384} // namespace
6385
6386void llvm::UpgradeFunctionAttributes(Function &F) {
6387 // If a function definition doesn't have the strictfp attribute,
6388 // convert any callsite strictfp attributes to nobuiltin.
6389 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
6390 StrictFPUpgradeVisitor SFPV;
6391 SFPV.visit(F);
6392 }
6393
6394 // Remove all incompatibile attributes from function.
6395 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
6396 F.getReturnType(), F.getAttributes().getRetAttrs()));
6397 for (auto &Arg : F.args())
6398 Arg.removeAttrs(
6399 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
6400
6401 bool AddingAttrs = false, RemovingAttrs = false;
6402 AttrBuilder AttrsToAdd(F.getContext());
6403 AttributeMask AttrsToRemove;
6404
6405 // Older versions of LLVM treated an "implicit-section-name" attribute
6406 // similarly to directly setting the section on a Function.
6407 if (Attribute A = F.getFnAttribute("implicit-section-name");
6408 A.isValid() && A.isStringAttribute()) {
6409 F.setSection(A.getValueAsString());
6410 AttrsToRemove.addAttribute("implicit-section-name");
6411 RemovingAttrs = true;
6412 }
6413
6414 if (Attribute A = F.getFnAttribute("nooutline");
6415 A.isValid() && A.isStringAttribute()) {
6416 AttrsToRemove.addAttribute("nooutline");
6417 AttrsToAdd.addAttribute(Attribute::NoOutline);
6418 AddingAttrs = RemovingAttrs = true;
6419 }
6420
6421 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
6422 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
6423 AttrsToRemove.addAttribute("uniform-work-group-size");
6424 RemovingAttrs = true;
6425 if (A.getValueAsString() == "true") {
6426 AttrsToAdd.addAttribute("uniform-work-group-size");
6427 AddingAttrs = true;
6428 }
6429 }
6430
6431 if (!F.empty()) {
6432 // For some reason this is called twice, and the first time is before any
6433 // instructions are loaded into the body.
6434
6435 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
6436 A.isValid()) {
6437
6438 if (A.getValueAsBool()) {
6439 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
6440 Visitor.visit(F);
6441 }
6442
6443 // We will leave behind dead attribute uses on external declarations, but
6444 // clang never added these to declarations anyway.
6445 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
6446 RemovingAttrs = true;
6447 }
6448 }
6449
6450 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
6451 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
6452
6453 bool HandleDenormalMode = false;
6454
6455 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
6456 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6457 if (ParsedMode.isValid()) {
6458 DenormalFPMath = ParsedMode;
6459 AttrsToRemove.addAttribute("denormal-fp-math");
6460 AddingAttrs = RemovingAttrs = true;
6461 HandleDenormalMode = true;
6462 }
6463 }
6464
6465 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
6466 Attr.isValid()) {
6467 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6468 if (ParsedMode.isValid()) {
6469 DenormalFPMathF32 = ParsedMode;
6470 AttrsToRemove.addAttribute("denormal-fp-math-f32");
6471 AddingAttrs = RemovingAttrs = true;
6472 HandleDenormalMode = true;
6473 }
6474 }
6475
6476 if (HandleDenormalMode)
6477 AttrsToAdd.addDenormalFPEnvAttr(
6478 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
6479
6480 if (RemovingAttrs)
6481 F.removeFnAttrs(AttrsToRemove);
6482
6483 if (AddingAttrs)
6484 F.addFnAttrs(AttrsToAdd);
6485}
6486
6487// Check if the function attribute is not present and set it.
6488static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
6489 StringRef Value) {
6490 if (!F.hasFnAttribute(FnAttrName))
6491 F.addFnAttr(FnAttrName, Value);
6492}
6493
6494// Check if the function attribute is not present and set it if needed.
6495// If the attribute is "false" then removes it.
6496// If the attribute is "true" resets it to a valueless attribute.
6497static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
6498 if (!F.hasFnAttribute(FnAttrName)) {
6499 if (Set)
6500 F.addFnAttr(FnAttrName);
6501 } else {
6502 auto A = F.getFnAttribute(FnAttrName);
6503 if ("false" == A.getValueAsString())
6504 F.removeFnAttr(FnAttrName);
6505 else if ("true" == A.getValueAsString()) {
6506 F.removeFnAttr(FnAttrName);
6507 F.addFnAttr(FnAttrName);
6508 }
6509 }
6510}
6511
6512void llvm::copyModuleAttrToFunctions(Module &M) {
6513 Triple T(M.getTargetTriple());
6514 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
6515 return;
6516
6517 uint64_t BTEValue = 0;
6518 uint64_t BPPLRValue = 0;
6519 uint64_t GCSValue = 0;
6520 uint64_t SRAValue = 0;
6521 uint64_t SRAALLValue = 0;
6522 uint64_t SRABKeyValue = 0;
6523
6524 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6525 if (ModFlags) {
6526 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6527 MDNode *Op = ModFlags->getOperand(I);
6528 if (Op->getNumOperands() != 3)
6529 continue;
6530
6531 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6532 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
6533 if (!ID || !CI)
6534 continue;
6535
6536 StringRef IDStr = ID->getString();
6537 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
6538 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
6539 : IDStr == "guarded-control-stack" ? &GCSValue
6540 : IDStr == "sign-return-address" ? &SRAValue
6541 : IDStr == "sign-return-address-all" ? &SRAALLValue
6542 : IDStr == "sign-return-address-with-bkey"
6543 ? &SRABKeyValue
6544 : nullptr;
6545 if (!ValPtr)
6546 continue;
6547
6548 *ValPtr = CI->getZExtValue();
6549 if (*ValPtr == 2)
6550 return;
6551 }
6552 }
6553
6554 bool BTE = BTEValue == 1;
6555 bool BPPLR = BPPLRValue == 1;
6556 bool GCS = GCSValue == 1;
6557 bool SRA = SRAValue == 1;
6558
6559 StringRef SignTypeValue = "non-leaf";
6560 if (SRA && SRAALLValue == 1)
6561 SignTypeValue = "all";
6562
6563 StringRef SignKeyValue = "a_key";
6564 if (SRA && SRABKeyValue == 1)
6565 SignKeyValue = "b_key";
6566
6567 for (Function &F : M.getFunctionList()) {
6568 if (F.isDeclaration())
6569 continue;
6570
6571 if (SRA) {
6572 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
6573 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
6574 } else {
6575 if (auto A = F.getFnAttribute("sign-return-address");
6576 A.isValid() && "none" == A.getValueAsString()) {
6577 F.removeFnAttr("sign-return-address");
6578 F.removeFnAttr("sign-return-address-key");
6579 }
6580 }
6581 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
6582 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
6583 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
6584 }
6585
6586 if (BTE)
6587 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
6588 if (BPPLR)
6589 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
6590 if (GCS)
6591 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
6592 if (SRA) {
6593 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
6594 if (SRAALLValue == 1)
6595 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
6596 if (SRABKeyValue == 1)
6597 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
6598 }
6599}
6600
6601static bool isOldLoopArgument(Metadata *MD) {
6602 auto *T = dyn_cast_or_null<MDTuple>(MD);
6603 if (!T)
6604 return false;
6605 if (T->getNumOperands() < 1)
6606 return false;
6607 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
6608 if (!S)
6609 return false;
6610 return S->getString().starts_with("llvm.vectorizer.");
6611}
6612
6613static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
6614 StringRef OldPrefix = "llvm.vectorizer.";
6615 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
6616
6617 if (OldTag == "llvm.vectorizer.unroll")
6618 return MDString::get(C, "llvm.loop.interleave.count");
6619
6620 return MDString::get(
6621 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
6622 .str());
6623}
6624
6625static Metadata *upgradeLoopArgument(Metadata *MD) {
6626 auto *T = dyn_cast_or_null<MDTuple>(MD);
6627 if (!T)
6628 return MD;
6629 if (T->getNumOperands() < 1)
6630 return MD;
6631 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
6632 if (!OldTag)
6633 return MD;
6634 if (!OldTag->getString().starts_with("llvm.vectorizer."))
6635 return MD;
6636
6637 // This has an old tag. Upgrade it.
6638 SmallVector<Metadata *, 8> Ops;
6639 Ops.reserve(T->getNumOperands());
6640 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
6641 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
6642 Ops.push_back(T->getOperand(I));
6643
6644 return MDTuple::get(T->getContext(), Ops);
6645}
6646
6647MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
6648 auto *T = dyn_cast<MDTuple>(&N);
6649 if (!T)
6650 return &N;
6651
6652 if (none_of(T->operands(), isOldLoopArgument))
6653 return &N;
6654
6655 SmallVector<Metadata *, 8> Ops;
6656 Ops.reserve(T->getNumOperands());
6657 for (Metadata *MD : T->operands())
6658 Ops.push_back(upgradeLoopArgument(MD));
6659
6660 return MDTuple::get(T->getContext(), Ops);
6661}
6662
6663std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
6664 Triple T(TT);
6665 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
6666 // the address space of globals to 1. This does not apply to SPIRV Logical.
6667 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
6668 !DL.contains("-G") && !DL.starts_with("G")) {
6669 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
6670 }
6671
6672 if (T.isLoongArch64() || T.isRISCV64()) {
6673 // Make i32 a native type for 64-bit LoongArch and RISC-V.
6674 auto I = DL.find("-n64-");
6675 if (I != StringRef::npos)
6676 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
6677 return DL.str();
6678 }
6679
6680 // AMDGPU data layout upgrades.
6681 std::string Res = DL.str();
6682 if (T.isAMDGPU()) {
6683 // Define address spaces for constants.
6684 if (!DL.contains("-G") && !DL.starts_with("G"))
6685 Res.append(Res.empty() ? "G1" : "-G1");
6686
6687 // AMDGCN data layout upgrades.
6688 if (T.isAMDGCN()) {
6689
6690 // Add missing non-integral declarations.
6691 // This goes before adding new address spaces to prevent incoherent string
6692 // values.
6693 if (!DL.contains("-ni") && !DL.starts_with("ni"))
6694 Res.append("-ni:7:8:9");
6695 // Update ni:7 to ni:7:8:9.
6696 if (DL.ends_with("ni:7"))
6697 Res.append(":8:9");
6698 if (DL.ends_with("ni:7:8"))
6699 Res.append(":9");
6700
6701 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
6702 // resources) An empty data layout has already been upgraded to G1 by now.
6703 if (!DL.contains("-p7") && !DL.starts_with("p7"))
6704 Res.append("-p7:160:256:256:32");
6705 if (!DL.contains("-p8") && !DL.starts_with("p8"))
6706 Res.append("-p8:128:128:128:48");
6707 constexpr StringRef OldP8("-p8:128:128-");
6708 if (DL.contains(OldP8))
6709 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
6710 if (!DL.contains("-p9") && !DL.starts_with("p9"))
6711 Res.append("-p9:192:256:256:32");
6712 }
6713
6714 // Upgrade the ELF mangling mode.
6715 if (!DL.contains("m:e"))
6716 Res = Res.empty() ? "m:e" : "m:e-" + Res;
6717
6718 return Res;
6719 }
6720
6721 if (T.isSystemZ() && !DL.empty()) {
6722 // Make sure the stack alignment is present.
6723 if (!DL.contains("-S64"))
6724 return "E-S64" + DL.drop_front(1).str();
6725 return DL.str();
6726 }
6727
6728 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
6729 // If the datalayout matches the expected format, add pointer size address
6730 // spaces to the datalayout.
6731 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
6732 if (!DL.contains(AddrSpaces)) {
6733 SmallVector<StringRef, 4> Groups;
6734 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
6735 if (R.match(Res, &Groups))
6736 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
6737 }
6738 };
6739
6740 // AArch64 data layout upgrades.
6741 if (T.isAArch64()) {
6742 // Add "-Fn32"
6743 if (!DL.empty() && !DL.contains("-Fn32"))
6744 Res.append("-Fn32");
6745 AddPtr32Ptr64AddrSpaces();
6746 return Res;
6747 }
6748
6749 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
6750 T.isWasm()) {
6751 // Mips64 with o32 ABI did not add "-i128:128".
6752 // Add "-i128:128"
6753 std::string I64 = "-i64:64";
6754 std::string I128 = "-i128:128";
6755 if (!StringRef(Res).contains(I128)) {
6756 size_t Pos = Res.find(I64);
6757 if (Pos != size_t(-1))
6758 Res.insert(Pos + I64.size(), I128);
6759 }
6760 }
6761
6762 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
6763 size_t Pos = Res.find("-S128");
6764 if (Pos == StringRef::npos)
6765 Pos = Res.size();
6766 Res.insert(Pos, "-f64:32:64");
6767 }
6768
6769 if (!T.isX86())
6770 return Res;
6771
6772 AddPtr32Ptr64AddrSpaces();
6773
6774 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
6775 // for i128 operations prior to this being reflected in the data layout, and
6776 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
6777 // boundaries, so although this is a breaking change, the upgrade is expected
6778 // to fix more IR than it breaks.
6779 // Intel MCU is an exception and uses 4-byte-alignment.
6780 if (!T.isOSIAMCU()) {
6781 std::string I128 = "-i128:128";
6782 if (StringRef Ref = Res; !Ref.contains(I128)) {
6783 SmallVector<StringRef, 4> Groups;
6784 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
6785 if (R.match(Res, &Groups))
6786 Res = (Groups[1] + I128 + Groups[3]).str();
6787 }
6788 }
6789
6790 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
6791 // Raising the alignment is safe because Clang did not produce f80 values in
6792 // the MSVC environment before this upgrade was added.
6793 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
6794 StringRef Ref = Res;
6795 auto I = Ref.find("-f80:32-");
6796 if (I != StringRef::npos)
6797 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
6798 }
6799
6800 return Res;
6801}
6802
6803void llvm::UpgradeAttributes(AttrBuilder &B) {
6804 StringRef FramePointer;
6805 Attribute A = B.getAttribute("no-frame-pointer-elim");
6806 if (A.isValid()) {
6807 // The value can be "true" or "false".
6808 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
6809 B.removeAttribute("no-frame-pointer-elim");
6810 }
6811 if (B.contains("no-frame-pointer-elim-non-leaf")) {
6812 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
6813 if (FramePointer != "all")
6814 FramePointer = "non-leaf";
6815 B.removeAttribute("no-frame-pointer-elim-non-leaf");
6816 }
6817 if (!FramePointer.empty())
6818 B.addAttribute("frame-pointer", FramePointer);
6819
6820 A = B.getAttribute("null-pointer-is-valid");
6821 if (A.isValid()) {
6822 // The value can be "true" or "false".
6823 bool NullPointerIsValid = A.getValueAsString() == "true";
6824 B.removeAttribute("null-pointer-is-valid");
6825 if (NullPointerIsValid)
6826 B.addAttribute(Attribute::NullPointerIsValid);
6827 }
6828
6829 A = B.getAttribute("uniform-work-group-size");
6830 if (A.isValid()) {
6831 StringRef Val = A.getValueAsString();
6832 if (!Val.empty()) {
6833 bool IsTrue = Val == "true";
6834 B.removeAttribute("uniform-work-group-size");
6835 if (IsTrue)
6836 B.addAttribute("uniform-work-group-size");
6837 }
6838 }
6839}
6840
6841void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
6842 // clang.arc.attachedcall bundles are now required to have an operand.
6843 // If they don't, it's okay to drop them entirely: when there is an operand,
6844 // the "attachedcall" is meaningful and required, but without an operand,
6845 // it's just a marker NOP. Dropping it merely prevents an optimization.
6846 erase_if(Bundles, [&](OperandBundleDef &OBD) {
6847 return OBD.getTag() == "clang.arc.attachedcall" &&
6848 OBD.inputs().empty();
6849 });
6850}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2114
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:6625
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:5918
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1281
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:1954
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1164
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6024
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2215
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2441
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1261
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2189
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:123
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:6488
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1189
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:5922
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:4878
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:6613
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:5885
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:110
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2069
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2326
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2040
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4550
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2427
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:544
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2366
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1255
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:177
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:1988
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1117
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:6601
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4607
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:94
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4948
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4697
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2305
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:4890
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:157
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:782
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2020
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2052
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2254
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:65
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:78
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:167
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2388
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2174
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2698
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2420
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2450
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:4900
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:6497
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2336
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:69
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2287
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4961
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:141
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2862
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3391
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
getNumElements
static unsigned getNumElements(Type *Ty)
Definition SLPVectorizer.cpp:329
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:483
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:81
LHS
Value * LHS
Definition X86PartialReduction.cpp:80
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:710
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:870
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:722
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:747
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:786
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:790
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:778
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:758
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:754
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:782
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:917
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:504
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:352
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1531
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1651
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3084
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3367
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1802
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1424
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2462
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2394
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2448
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1968
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1491
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1651
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:419
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:363
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3453
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2311
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:215
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:213
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:241
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:869
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:168
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:211
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:246
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:166
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:449
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:901
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:216
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:886
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:548
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:298
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:629
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2858
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:546
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:86
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:112
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1751
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:714
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1504
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:94
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:308
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:181
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1080
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1444
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1572
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1450
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1244
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:614
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1529
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:110
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1760
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1535
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1527
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1523
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1542
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1856
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1533
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2026
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:891
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2031
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:387
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:533
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:691
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1225
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:297
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:882
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:730
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:591
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:629
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:844
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:479
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:47
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:839
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5229
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:393
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:552
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:712
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:318
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:399
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:692
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:721
uint32_t
uint64_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2356
Changed
Changed
Definition ObjCARCOpts.cpp:2366
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1292
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:779
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1273
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1266
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:558
llvm::Length
@ Length
Definition DWP.cpp:558
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:4980
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:6313
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1668
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:2689
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::Int32Ty
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
Definition InstrProf.h:328
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:1902
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:6647
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:6803
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:5750
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:5983
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6170
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:284
llvm::copyModuleAttrToFunctions
void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:6512
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:6841
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:5817
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::Version
FunctionAddr VTableAddr uintptr_t uintptr_t Version
Definition InstrProf.h:334
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:279
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:6663
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1752
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:1921
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:904
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:704
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:486
llvm::Data
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:221
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:5795
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1771
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2191
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:5839
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:6386
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:5769
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6048
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:8145
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:876
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0