LLVM 17.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/StringSwitch.h"
17#include "llvm/BinaryFormat/Dwarf.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DebugInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21#include "llvm/IR/DiagnosticInfo.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/InstVisitor.h"
25#include "llvm/IR/Instruction.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/IntrinsicsAArch64.h"
29#include "llvm/IR/IntrinsicsARM.h"
30#include "llvm/IR/IntrinsicsWebAssembly.h"
31#include "llvm/IR/IntrinsicsX86.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/Metadata.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/Verifier.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/Regex.h"
39#include "llvm/TargetParser/Triple.h"
40#include <cstring>
41
42using namespace llvm;
43
44static cl::opt<bool>
45 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
46 cl::desc("Disable autoupgrade of debug info"));
47
48static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
49
50// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
51// changed their type from v4f32 to v2i64.
52static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
53 Function *&NewFn) {
54 // Check whether this is an old version of the function, which received
55 // v4f32 arguments.
56 Type *Arg0Type = F->getFunctionType()->getParamType(0);
57 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
58 return false;
59
60 // Yes, it's old, replace it with new version.
61 rename(F);
62 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
63 return true;
64}
65
66// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
67// arguments have changed their type from i32 to i8.
68static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
69 Function *&NewFn) {
70 // Check that the last argument is an i32.
71 Type *LastArgType = F->getFunctionType()->getParamType(
72 F->getFunctionType()->getNumParams() - 1);
73 if (!LastArgType->isIntegerTy(32))
74 return false;
75
76 // Move this function aside and map down.
77 rename(F);
78 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
79 return true;
80}
81
82// Upgrade the declaration of fp compare intrinsics that change return type
83// from scalar to vXi1 mask.
84static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
85 Function *&NewFn) {
86 // Check if the return type is a vector.
87 if (F->getReturnType()->isVectorTy())
88 return false;
89
90 rename(F);
91 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
92 return true;
93}
94
95static bool UpgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
96 Function *&NewFn) {
97 if (F->getReturnType()->getScalarType()->isBFloatTy())
98 return false;
99
100 rename(F);
101 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
102 return true;
103}
104
105static bool UpgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
106 Function *&NewFn) {
107 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
108 return false;
109
110 rename(F);
111 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
112 return true;
113}
114
115static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
116 // All of the intrinsics matches below should be marked with which llvm
117 // version started autoupgrading them. At some point in the future we would
118 // like to use this information to remove upgrade code for some older
119 // intrinsics. It is currently undecided how we will determine that future
120 // point.
121 if (Name == "addcarryx.u32" || // Added in 8.0
122 Name == "addcarryx.u64" || // Added in 8.0
123 Name == "addcarry.u32" || // Added in 8.0
124 Name == "addcarry.u64" || // Added in 8.0
125 Name == "subborrow.u32" || // Added in 8.0
126 Name == "subborrow.u64" || // Added in 8.0
127 Name.startswith("sse2.padds.") || // Added in 8.0
128 Name.startswith("sse2.psubs.") || // Added in 8.0
129 Name.startswith("sse2.paddus.") || // Added in 8.0
130 Name.startswith("sse2.psubus.") || // Added in 8.0
131 Name.startswith("avx2.padds.") || // Added in 8.0
132 Name.startswith("avx2.psubs.") || // Added in 8.0
133 Name.startswith("avx2.paddus.") || // Added in 8.0
134 Name.startswith("avx2.psubus.") || // Added in 8.0
135 Name.startswith("avx512.padds.") || // Added in 8.0
136 Name.startswith("avx512.psubs.") || // Added in 8.0
137 Name.startswith("avx512.mask.padds.") || // Added in 8.0
138 Name.startswith("avx512.mask.psubs.") || // Added in 8.0
139 Name.startswith("avx512.mask.paddus.") || // Added in 8.0
140 Name.startswith("avx512.mask.psubus.") || // Added in 8.0
141 Name=="ssse3.pabs.b.128" || // Added in 6.0
142 Name=="ssse3.pabs.w.128" || // Added in 6.0
143 Name=="ssse3.pabs.d.128" || // Added in 6.0
144 Name.startswith("fma4.vfmadd.s") || // Added in 7.0
145 Name.startswith("fma.vfmadd.") || // Added in 7.0
146 Name.startswith("fma.vfmsub.") || // Added in 7.0
147 Name.startswith("fma.vfmsubadd.") || // Added in 7.0
148 Name.startswith("fma.vfnmadd.") || // Added in 7.0
149 Name.startswith("fma.vfnmsub.") || // Added in 7.0
150 Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
151 Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
152 Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
153 Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
154 Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
155 Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
156 Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
157 Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
158 Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
159 Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
160 Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
161 Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
162 Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
163 Name.startswith("avx512.kunpck") || //added in 6.0
164 Name.startswith("avx2.pabs.") || // Added in 6.0
165 Name.startswith("avx512.mask.pabs.") || // Added in 6.0
166 Name.startswith("avx512.broadcastm") || // Added in 6.0
167 Name == "sse.sqrt.ss" || // Added in 7.0
168 Name == "sse2.sqrt.sd" || // Added in 7.0
169 Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
170 Name.startswith("avx.sqrt.p") || // Added in 7.0
171 Name.startswith("sse2.sqrt.p") || // Added in 7.0
172 Name.startswith("sse.sqrt.p") || // Added in 7.0
173 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
174 Name.startswith("sse2.pcmpeq.") || // Added in 3.1
175 Name.startswith("sse2.pcmpgt.") || // Added in 3.1
176 Name.startswith("avx2.pcmpeq.") || // Added in 3.1
177 Name.startswith("avx2.pcmpgt.") || // Added in 3.1
178 Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
179 Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
180 Name.startswith("avx.vperm2f128.") || // Added in 6.0
181 Name == "avx2.vperm2i128" || // Added in 6.0
182 Name == "sse.add.ss" || // Added in 4.0
183 Name == "sse2.add.sd" || // Added in 4.0
184 Name == "sse.sub.ss" || // Added in 4.0
185 Name == "sse2.sub.sd" || // Added in 4.0
186 Name == "sse.mul.ss" || // Added in 4.0
187 Name == "sse2.mul.sd" || // Added in 4.0
188 Name == "sse.div.ss" || // Added in 4.0
189 Name == "sse2.div.sd" || // Added in 4.0
190 Name == "sse41.pmaxsb" || // Added in 3.9
191 Name == "sse2.pmaxs.w" || // Added in 3.9
192 Name == "sse41.pmaxsd" || // Added in 3.9
193 Name == "sse2.pmaxu.b" || // Added in 3.9
194 Name == "sse41.pmaxuw" || // Added in 3.9
195 Name == "sse41.pmaxud" || // Added in 3.9
196 Name == "sse41.pminsb" || // Added in 3.9
197 Name == "sse2.pmins.w" || // Added in 3.9
198 Name == "sse41.pminsd" || // Added in 3.9
199 Name == "sse2.pminu.b" || // Added in 3.9
200 Name == "sse41.pminuw" || // Added in 3.9
201 Name == "sse41.pminud" || // Added in 3.9
202 Name == "avx512.kand.w" || // Added in 7.0
203 Name == "avx512.kandn.w" || // Added in 7.0
204 Name == "avx512.knot.w" || // Added in 7.0
205 Name == "avx512.kor.w" || // Added in 7.0
206 Name == "avx512.kxor.w" || // Added in 7.0
207 Name == "avx512.kxnor.w" || // Added in 7.0
208 Name == "avx512.kortestc.w" || // Added in 7.0
209 Name == "avx512.kortestz.w" || // Added in 7.0
210 Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
211 Name.startswith("avx2.pmax") || // Added in 3.9
212 Name.startswith("avx2.pmin") || // Added in 3.9
213 Name.startswith("avx512.mask.pmax") || // Added in 4.0
214 Name.startswith("avx512.mask.pmin") || // Added in 4.0
215 Name.startswith("avx2.vbroadcast") || // Added in 3.8
216 Name.startswith("avx2.pbroadcast") || // Added in 3.8
217 Name.startswith("avx.vpermil.") || // Added in 3.1
218 Name.startswith("sse2.pshuf") || // Added in 3.9
219 Name.startswith("avx512.pbroadcast") || // Added in 3.9
220 Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
221 Name.startswith("avx512.mask.movddup") || // Added in 3.9
222 Name.startswith("avx512.mask.movshdup") || // Added in 3.9
223 Name.startswith("avx512.mask.movsldup") || // Added in 3.9
224 Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
225 Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
226 Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
227 Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
228 Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
229 Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
230 Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
231 Name.startswith("avx512.mask.punpckl") || // Added in 3.9
232 Name.startswith("avx512.mask.punpckh") || // Added in 3.9
233 Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
234 Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
235 Name.startswith("avx512.mask.pand.") || // Added in 3.9
236 Name.startswith("avx512.mask.pandn.") || // Added in 3.9
237 Name.startswith("avx512.mask.por.") || // Added in 3.9
238 Name.startswith("avx512.mask.pxor.") || // Added in 3.9
239 Name.startswith("avx512.mask.and.") || // Added in 3.9
240 Name.startswith("avx512.mask.andn.") || // Added in 3.9
241 Name.startswith("avx512.mask.or.") || // Added in 3.9
242 Name.startswith("avx512.mask.xor.") || // Added in 3.9
243 Name.startswith("avx512.mask.padd.") || // Added in 4.0
244 Name.startswith("avx512.mask.psub.") || // Added in 4.0
245 Name.startswith("avx512.mask.pmull.") || // Added in 4.0
246 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
247 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
248 Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
249 Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
250 Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
251 Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
252 Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
253 Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
254 Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
255 Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
256 Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
257 Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
258 Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
259 Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
260 Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
261 Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
262 Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
263 Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
264 Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
265 Name == "avx512.cvtusi2sd" || // Added in 7.0
266 Name.startswith("avx512.mask.permvar.") || // Added in 7.0
267 Name == "sse2.pmulu.dq" || // Added in 7.0
268 Name == "sse41.pmuldq" || // Added in 7.0
269 Name == "avx2.pmulu.dq" || // Added in 7.0
270 Name == "avx2.pmul.dq" || // Added in 7.0
271 Name == "avx512.pmulu.dq.512" || // Added in 7.0
272 Name == "avx512.pmul.dq.512" || // Added in 7.0
273 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
274 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
275 Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
276 Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
277 Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
278 Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
279 Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
280 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
281 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
282 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
283 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
284 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
285 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
286 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
287 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
288 Name.startswith("avx512.cmp.p") || // Added in 12.0
289 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
290 Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
291 Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
292 Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
293 Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
294 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
295 Name.startswith("avx512.mask.psll.d") || // Added in 4.0
296 Name.startswith("avx512.mask.psll.q") || // Added in 4.0
297 Name.startswith("avx512.mask.psll.w") || // Added in 4.0
298 Name.startswith("avx512.mask.psra.d") || // Added in 4.0
299 Name.startswith("avx512.mask.psra.q") || // Added in 4.0
300 Name.startswith("avx512.mask.psra.w") || // Added in 4.0
301 Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
302 Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
303 Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
304 Name.startswith("avx512.mask.pslli") || // Added in 4.0
305 Name.startswith("avx512.mask.psrai") || // Added in 4.0
306 Name.startswith("avx512.mask.psrli") || // Added in 4.0
307 Name.startswith("avx512.mask.psllv") || // Added in 4.0
308 Name.startswith("avx512.mask.psrav") || // Added in 4.0
309 Name.startswith("avx512.mask.psrlv") || // Added in 4.0
310 Name.startswith("sse41.pmovsx") || // Added in 3.8
311 Name.startswith("sse41.pmovzx") || // Added in 3.9
312 Name.startswith("avx2.pmovsx") || // Added in 3.9
313 Name.startswith("avx2.pmovzx") || // Added in 3.9
314 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
315 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
316 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
317 Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
318 Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
319 Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
320 Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
321 Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
322 Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
323 Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
324 Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
325 Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
326 Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
327 Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
328 Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
329 Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
330 Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
331 Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
332 Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
333 Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
334 Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
335 Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
336 Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
337 Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
338 Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
339 Name.startswith("avx512.vpshld.") || // Added in 8.0
340 Name.startswith("avx512.vpshrd.") || // Added in 8.0
341 Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
342 Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
343 Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
344 Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
345 Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
346 Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
347 Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
348 Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
349 Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
350 Name.startswith("avx512.mask.conflict.") || // Added in 9.0
351 Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
352 Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
353 Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
354 Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
355 Name == "sse.cvtsi2ss" || // Added in 7.0
356 Name == "sse.cvtsi642ss" || // Added in 7.0
357 Name == "sse2.cvtsi2sd" || // Added in 7.0
358 Name == "sse2.cvtsi642sd" || // Added in 7.0
359 Name == "sse2.cvtss2sd" || // Added in 7.0
360 Name == "sse2.cvtdq2pd" || // Added in 3.9
361 Name == "sse2.cvtdq2ps" || // Added in 7.0
362 Name == "sse2.cvtps2pd" || // Added in 3.9
363 Name == "avx.cvtdq2.pd.256" || // Added in 3.9
364 Name == "avx.cvtdq2.ps.256" || // Added in 7.0
365 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
366 Name.startswith("vcvtph2ps.") || // Added in 11.0
367 Name.startswith("avx.vinsertf128.") || // Added in 3.7
368 Name == "avx2.vinserti128" || // Added in 3.7
369 Name.startswith("avx512.mask.insert") || // Added in 4.0
370 Name.startswith("avx.vextractf128.") || // Added in 3.7
371 Name == "avx2.vextracti128" || // Added in 3.7
372 Name.startswith("avx512.mask.vextract") || // Added in 4.0
373 Name.startswith("sse4a.movnt.") || // Added in 3.9
374 Name.startswith("avx.movnt.") || // Added in 3.2
375 Name.startswith("avx512.storent.") || // Added in 3.9
376 Name == "sse41.movntdqa" || // Added in 5.0
377 Name == "avx2.movntdqa" || // Added in 5.0
378 Name == "avx512.movntdqa" || // Added in 5.0
379 Name == "sse2.storel.dq" || // Added in 3.9
380 Name.startswith("sse.storeu.") || // Added in 3.9
381 Name.startswith("sse2.storeu.") || // Added in 3.9
382 Name.startswith("avx.storeu.") || // Added in 3.9
383 Name.startswith("avx512.mask.storeu.") || // Added in 3.9
384 Name.startswith("avx512.mask.store.p") || // Added in 3.9
385 Name.startswith("avx512.mask.store.b.") || // Added in 3.9
386 Name.startswith("avx512.mask.store.w.") || // Added in 3.9
387 Name.startswith("avx512.mask.store.d.") || // Added in 3.9
388 Name.startswith("avx512.mask.store.q.") || // Added in 3.9
389 Name == "avx512.mask.store.ss" || // Added in 7.0
390 Name.startswith("avx512.mask.loadu.") || // Added in 3.9
391 Name.startswith("avx512.mask.load.") || // Added in 3.9
392 Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
393 Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
394 Name.startswith("avx512.mask.expand.b") || // Added in 9.0
395 Name.startswith("avx512.mask.expand.w") || // Added in 9.0
396 Name.startswith("avx512.mask.expand.d") || // Added in 9.0
397 Name.startswith("avx512.mask.expand.q") || // Added in 9.0
398 Name.startswith("avx512.mask.expand.p") || // Added in 9.0
399 Name.startswith("avx512.mask.compress.b") || // Added in 9.0
400 Name.startswith("avx512.mask.compress.w") || // Added in 9.0
401 Name.startswith("avx512.mask.compress.d") || // Added in 9.0
402 Name.startswith("avx512.mask.compress.q") || // Added in 9.0
403 Name.startswith("avx512.mask.compress.p") || // Added in 9.0
404 Name == "sse42.crc32.64.8" || // Added in 3.4
405 Name.startswith("avx.vbroadcast.s") || // Added in 3.5
406 Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
407 Name.startswith("avx512.mask.palignr.") || // Added in 3.9
408 Name.startswith("avx512.mask.valign.") || // Added in 4.0
409 Name.startswith("sse2.psll.dq") || // Added in 3.7
410 Name.startswith("sse2.psrl.dq") || // Added in 3.7
411 Name.startswith("avx2.psll.dq") || // Added in 3.7
412 Name.startswith("avx2.psrl.dq") || // Added in 3.7
413 Name.startswith("avx512.psll.dq") || // Added in 3.9
414 Name.startswith("avx512.psrl.dq") || // Added in 3.9
415 Name == "sse41.pblendw" || // Added in 3.7
416 Name.startswith("sse41.blendp") || // Added in 3.7
417 Name.startswith("avx.blend.p") || // Added in 3.7
418 Name == "avx2.pblendw" || // Added in 3.7
419 Name.startswith("avx2.pblendd.") || // Added in 3.7
420 Name.startswith("avx.vbroadcastf128") || // Added in 4.0
421 Name == "avx2.vbroadcasti128" || // Added in 3.7
422 Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
423 Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
424 Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
425 Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
426 Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
427 Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
428 Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
429 Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
430 Name == "xop.vpcmov" || // Added in 3.8
431 Name == "xop.vpcmov.256" || // Added in 5.0
432 Name.startswith("avx512.mask.move.s") || // Added in 4.0
433 Name.startswith("avx512.cvtmask2") || // Added in 5.0
434 Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
435 Name.startswith("xop.vprot") || // Added in 8.0
436 Name.startswith("avx512.prol") || // Added in 8.0
437 Name.startswith("avx512.pror") || // Added in 8.0
438 Name.startswith("avx512.mask.prorv.") || // Added in 8.0
439 Name.startswith("avx512.mask.pror.") || // Added in 8.0
440 Name.startswith("avx512.mask.prolv.") || // Added in 8.0
441 Name.startswith("avx512.mask.prol.") || // Added in 8.0
442 Name.startswith("avx512.ptestm") || //Added in 6.0
443 Name.startswith("avx512.ptestnm") || //Added in 6.0
444 Name.startswith("avx512.mask.pavg")) // Added in 6.0
445 return true;
446
447 return false;
448}
449
450static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
451 Function *&NewFn) {
452 // Only handle intrinsics that start with "x86.".
453 if (!Name.startswith("x86."))
454 return false;
455 // Remove "x86." prefix.
456 Name = Name.substr(4);
457
458 if (ShouldUpgradeX86Intrinsic(F, Name)) {
459 NewFn = nullptr;
460 return true;
461 }
462
463 if (Name == "rdtscp") { // Added in 8.0
464 // If this intrinsic has 0 operands, it's the new version.
465 if (F->getFunctionType()->getNumParams() == 0)
466 return false;
467
468 rename(F);
469 NewFn = Intrinsic::getDeclaration(F->getParent(),
470 Intrinsic::x86_rdtscp);
471 return true;
472 }
473
474 // SSE4.1 ptest functions may have an old signature.
475 if (Name.startswith("sse41.ptest")) { // Added in 3.2
476 if (Name.substr(11) == "c")
477 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
478 if (Name.substr(11) == "z")
479 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
480 if (Name.substr(11) == "nzc")
481 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
482 }
483 // Several blend and other instructions with masks used the wrong number of
484 // bits.
485 if (Name == "sse41.insertps") // Added in 3.6
486 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
487 NewFn);
488 if (Name == "sse41.dppd") // Added in 3.6
489 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
490 NewFn);
491 if (Name == "sse41.dpps") // Added in 3.6
492 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
493 NewFn);
494 if (Name == "sse41.mpsadbw") // Added in 3.6
495 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
496 NewFn);
497 if (Name == "avx.dp.ps.256") // Added in 3.6
498 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
499 NewFn);
500 if (Name == "avx2.mpsadbw") // Added in 3.6
501 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
502 NewFn);
503 if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
504 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
505 NewFn);
506 if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
507 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
508 NewFn);
509 if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
510 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
511 NewFn);
512 if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
513 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
514 NewFn);
515 if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
516 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
517 NewFn);
518 if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
519 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
520 NewFn);
521 if (Name == "avx512bf16.cvtne2ps2bf16.128") // Added in 9.0
522 return UpgradeX86BF16Intrinsic(
523 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128, NewFn);
524 if (Name == "avx512bf16.cvtne2ps2bf16.256") // Added in 9.0
525 return UpgradeX86BF16Intrinsic(
526 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256, NewFn);
527 if (Name == "avx512bf16.cvtne2ps2bf16.512") // Added in 9.0
528 return UpgradeX86BF16Intrinsic(
529 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512, NewFn);
530 if (Name == "avx512bf16.mask.cvtneps2bf16.128") // Added in 9.0
531 return UpgradeX86BF16Intrinsic(
532 F, Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, NewFn);
533 if (Name == "avx512bf16.cvtneps2bf16.256") // Added in 9.0
534 return UpgradeX86BF16Intrinsic(
535 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_256, NewFn);
536 if (Name == "avx512bf16.cvtneps2bf16.512") // Added in 9.0
537 return UpgradeX86BF16Intrinsic(
538 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_512, NewFn);
539 if (Name == "avx512bf16.dpbf16ps.128") // Added in 9.0
540 return UpgradeX86BF16DPIntrinsic(
541 F, Intrinsic::x86_avx512bf16_dpbf16ps_128, NewFn);
542 if (Name == "avx512bf16.dpbf16ps.256") // Added in 9.0
543 return UpgradeX86BF16DPIntrinsic(
544 F, Intrinsic::x86_avx512bf16_dpbf16ps_256, NewFn);
545 if (Name == "avx512bf16.dpbf16ps.512") // Added in 9.0
546 return UpgradeX86BF16DPIntrinsic(
547 F, Intrinsic::x86_avx512bf16_dpbf16ps_512, NewFn);
548
549 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
550 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
551 rename(F);
552 NewFn = Intrinsic::getDeclaration(F->getParent(),
553 Intrinsic::x86_xop_vfrcz_ss);
554 return true;
555 }
556 if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
557 rename(F);
558 NewFn = Intrinsic::getDeclaration(F->getParent(),
559 Intrinsic::x86_xop_vfrcz_sd);
560 return true;
561 }
562 // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
563 if (Name.startswith("xop.vpermil2")) { // Added in 3.9
564 auto Idx = F->getFunctionType()->getParamType(2);
565 if (Idx->isFPOrFPVectorTy()) {
566 rename(F);
567 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
568 unsigned EltSize = Idx->getScalarSizeInBits();
569 Intrinsic::ID Permil2ID;
570 if (EltSize == 64 && IdxSize == 128)
571 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
572 else if (EltSize == 32 && IdxSize == 128)
573 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
574 else if (EltSize == 64 && IdxSize == 256)
575 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
576 else
577 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
578 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
579 return true;
580 }
581 }
582
583 if (Name == "seh.recoverfp") {
584 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
585 return true;
586 }
587
588 return false;
589}
590
591static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
592 assert(F && "Illegal to upgrade a non-existent Function.");
593
594 // Quickly eliminate it, if it's not a candidate.
595 StringRef Name = F->getName();
596 if (Name.size() <= 7 || !Name.startswith("llvm."))
597 return false;
598 Name = Name.substr(5); // Strip off "llvm."
599
600 switch (Name[0]) {
601 default: break;
602 case 'a': {
603 if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
604 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
605 F->arg_begin()->getType());
606 return true;
607 }
608 if (Name.startswith("aarch64.neon.frintn")) {
609 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
610 F->arg_begin()->getType());
611 return true;
612 }
613 if (Name.startswith("aarch64.neon.rbit")) {
614 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
615 F->arg_begin()->getType());
616 return true;
617 }
618 if (Name == "aarch64.sve.bfdot.lane") {
619 NewFn = Intrinsic::getDeclaration(F->getParent(),
620 Intrinsic::aarch64_sve_bfdot_lane_v2);
621 return true;
622 }
623 if (Name == "aarch64.sve.bfmlalb.lane") {
624 NewFn = Intrinsic::getDeclaration(F->getParent(),
625 Intrinsic::aarch64_sve_bfmlalb_lane_v2);
626 return true;
627 }
628 if (Name == "aarch64.sve.bfmlalt.lane") {
629 NewFn = Intrinsic::getDeclaration(F->getParent(),
630 Intrinsic::aarch64_sve_bfmlalt_lane_v2);
631 return true;
632 }
633 static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
634 if (LdRegex.match(Name)) {
635 Type *ScalarTy =
636 dyn_cast<VectorType>(F->getReturnType())->getElementType();
637 ElementCount EC =
638 dyn_cast<VectorType>(F->arg_begin()->getType())->getElementCount();
639 Type *Ty = VectorType::get(ScalarTy, EC);
640 Intrinsic::ID ID =
641 StringSwitch<Intrinsic::ID>(Name)
642 .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
643 .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
644 .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
645 .Default(Intrinsic::not_intrinsic);
646 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty);
647 return true;
648 }
649 if (Name.startswith("aarch64.sve.tuple.get")) {
650 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
651 NewFn = Intrinsic::getDeclaration(F->getParent(),
652 Intrinsic::vector_extract, Tys);
653 return true;
654 }
655 if (Name.startswith("aarch64.sve.tuple.set")) {
656 auto Args = F->getFunctionType()->params();
657 Type *Tys[] = {Args[0], Args[2], Args[1]};
658 NewFn = Intrinsic::getDeclaration(F->getParent(),
659 Intrinsic::vector_insert, Tys);
660 return true;
661 }
662 static const Regex CreateTupleRegex(
663 "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
664 if (CreateTupleRegex.match(Name)) {
665 auto Args = F->getFunctionType()->params();
666 Type *Tys[] = {F->getReturnType(), Args[1]};
667 NewFn = Intrinsic::getDeclaration(F->getParent(),
668 Intrinsic::vector_insert, Tys);
669 return true;
670 }
671 if (Name.startswith("arm.neon.vclz")) {
672 Type* args[2] = {
673 F->arg_begin()->getType(),
674 Type::getInt1Ty(F->getContext())
675 };
676 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
677 // the end of the name. Change name from llvm.arm.neon.vclz.* to
678 // llvm.ctlz.*
679 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
680 NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
681 "llvm.ctlz." + Name.substr(14), F->getParent());
682 return true;
683 }
684 if (Name.startswith("arm.neon.vcnt")) {
685 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
686 F->arg_begin()->getType());
687 return true;
688 }
689 static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
690 if (vstRegex.match(Name)) {
691 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
692 Intrinsic::arm_neon_vst2,
693 Intrinsic::arm_neon_vst3,
694 Intrinsic::arm_neon_vst4};
695
696 static const Intrinsic::ID StoreLaneInts[] = {
697 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
698 Intrinsic::arm_neon_vst4lane
699 };
700
701 auto fArgs = F->getFunctionType()->params();
702 Type *Tys[] = {fArgs[0], fArgs[1]};
703 if (!Name.contains("lane"))
704 NewFn = Intrinsic::getDeclaration(F->getParent(),
705 StoreInts[fArgs.size() - 3], Tys);
706 else
707 NewFn = Intrinsic::getDeclaration(F->getParent(),
708 StoreLaneInts[fArgs.size() - 5], Tys);
709 return true;
710 }
711 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
712 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
713 return true;
714 }
715 if (Name.startswith("arm.neon.vqadds.")) {
716 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
717 F->arg_begin()->getType());
718 return true;
719 }
720 if (Name.startswith("arm.neon.vqaddu.")) {
721 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
722 F->arg_begin()->getType());
723 return true;
724 }
725 if (Name.startswith("arm.neon.vqsubs.")) {
726 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
727 F->arg_begin()->getType());
728 return true;
729 }
730 if (Name.startswith("arm.neon.vqsubu.")) {
731 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
732 F->arg_begin()->getType());
733 return true;
734 }
735 if (Name.startswith("aarch64.neon.addp")) {
736 if (F->arg_size() != 2)
737 break; // Invalid IR.
738 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
739 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
740 NewFn = Intrinsic::getDeclaration(F->getParent(),
741 Intrinsic::aarch64_neon_faddp, Ty);
742 return true;
743 }
744 }
745
746 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
747 // respectively
748 if ((Name.startswith("arm.neon.bfdot.") ||
749 Name.startswith("aarch64.neon.bfdot.")) &&
750 Name.endswith("i8")) {
751 Intrinsic::ID IID =
752 StringSwitch<Intrinsic::ID>(Name)
753 .Cases("arm.neon.bfdot.v2f32.v8i8",
754 "arm.neon.bfdot.v4f32.v16i8",
755 Intrinsic::arm_neon_bfdot)
756 .Cases("aarch64.neon.bfdot.v2f32.v8i8",
757 "aarch64.neon.bfdot.v4f32.v16i8",
758 Intrinsic::aarch64_neon_bfdot)
759 .Default(Intrinsic::not_intrinsic);
760 if (IID == Intrinsic::not_intrinsic)
761 break;
762
763 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
764 assert((OperandWidth == 64 || OperandWidth == 128) &&
765 "Unexpected operand width");
766 LLVMContext &Ctx = F->getParent()->getContext();
767 std::array<Type *, 2> Tys {{
768 F->getReturnType(),
769 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
770 }};
771 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
772 return true;
773 }
774
775 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
776 // and accept v8bf16 instead of v16i8
777 if ((Name.startswith("arm.neon.bfm") ||
778 Name.startswith("aarch64.neon.bfm")) &&
779 Name.endswith(".v4f32.v16i8")) {
780 Intrinsic::ID IID =
781 StringSwitch<Intrinsic::ID>(Name)
782 .Case("arm.neon.bfmmla.v4f32.v16i8",
783 Intrinsic::arm_neon_bfmmla)
784 .Case("arm.neon.bfmlalb.v4f32.v16i8",
785 Intrinsic::arm_neon_bfmlalb)
786 .Case("arm.neon.bfmlalt.v4f32.v16i8",
787 Intrinsic::arm_neon_bfmlalt)
788 .Case("aarch64.neon.bfmmla.v4f32.v16i8",
789 Intrinsic::aarch64_neon_bfmmla)
790 .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
791 Intrinsic::aarch64_neon_bfmlalb)
792 .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
793 Intrinsic::aarch64_neon_bfmlalt)
794 .Default(Intrinsic::not_intrinsic);
795 if (IID == Intrinsic::not_intrinsic)
796 break;
797
798 std::array<Type *, 0> Tys;
799 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
800 return true;
801 }
802
803 if (Name == "arm.mve.vctp64" &&
804 cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
805 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the
806 // function and deal with it below in UpgradeIntrinsicCall.
807 rename(F);
808 return true;
809 }
810 // These too are changed to accept a v2i1 insteead of the old v4i1.
811 if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
812 Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
813 Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
814 Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
815 Name ==
816 "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
817 Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
818 Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
819 Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
820 Name ==
821 "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
822 Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
823 Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
824 Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
825 Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
826 Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
827 Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
828 Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
829 return true;
830
831 if (Name == "amdgcn.alignbit") {
832 // Target specific intrinsic became redundant
833 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr,
834 {F->getReturnType()});
835 return true;
836 }
837
838 break;
839 }
840
841 case 'c': {
842 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
843 rename(F);
844 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
845 F->arg_begin()->getType());
846 return true;
847 }
848 if (Name.startswith("cttz.") && F->arg_size() == 1) {
849 rename(F);
850 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
851 F->arg_begin()->getType());
852 return true;
853 }
854 break;
855 }
856 case 'd': {
857 if (Name == "dbg.addr") {
858 rename(F);
859 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
860 return true;
861 }
862 if (Name == "dbg.value" && F->arg_size() == 4) {
863 rename(F);
864 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
865 return true;
866 }
867 break;
868 }
869 case 'e': {
870 if (Name.startswith("experimental.vector.extract.")) {
871 rename(F);
872 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
873 NewFn = Intrinsic::getDeclaration(F->getParent(),
874 Intrinsic::vector_extract, Tys);
875 return true;
876 }
877
878 if (Name.startswith("experimental.vector.insert.")) {
879 rename(F);
880 auto Args = F->getFunctionType()->params();
881 Type *Tys[] = {Args[0], Args[1]};
882 NewFn = Intrinsic::getDeclaration(F->getParent(),
883 Intrinsic::vector_insert, Tys);
884 return true;
885 }
886
887 SmallVector<StringRef, 2> Groups;
888 static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
889 if (R.match(Name, &Groups)) {
890 Intrinsic::ID ID;
891 ID = StringSwitch<Intrinsic::ID>(Groups[1])
892 .Case("add", Intrinsic::vector_reduce_add)
893 .Case("mul", Intrinsic::vector_reduce_mul)
894 .Case("and", Intrinsic::vector_reduce_and)
895 .Case("or", Intrinsic::vector_reduce_or)
896 .Case("xor", Intrinsic::vector_reduce_xor)
897 .Case("smax", Intrinsic::vector_reduce_smax)
898 .Case("smin", Intrinsic::vector_reduce_smin)
899 .Case("umax", Intrinsic::vector_reduce_umax)
900 .Case("umin", Intrinsic::vector_reduce_umin)
901 .Case("fmax", Intrinsic::vector_reduce_fmax)
902 .Case("fmin", Intrinsic::vector_reduce_fmin)
903 .Default(Intrinsic::not_intrinsic);
904 if (ID != Intrinsic::not_intrinsic) {
905 rename(F);
906 auto Args = F->getFunctionType()->params();
907 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
908 return true;
909 }
910 }
911 static const Regex R2(
912 "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
913 Groups.clear();
914 if (R2.match(Name, &Groups)) {
915 Intrinsic::ID ID = Intrinsic::not_intrinsic;
916 if (Groups[1] == "fadd")
917 ID = Intrinsic::vector_reduce_fadd;
918 if (Groups[1] == "fmul")
919 ID = Intrinsic::vector_reduce_fmul;
920 if (ID != Intrinsic::not_intrinsic) {
921 rename(F);
922 auto Args = F->getFunctionType()->params();
923 Type *Tys[] = {Args[1]};
924 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
925 return true;
926 }
927 }
928 break;
929 }
930 case 'f':
931 if (Name.startswith("flt.rounds")) {
932 rename(F);
933 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding);
934 return true;
935 }
936 break;
937 case 'i':
938 case 'l': {
939 bool IsLifetimeStart = Name.startswith("lifetime.start");
940 if (IsLifetimeStart || Name.startswith("invariant.start")) {
941 Intrinsic::ID ID = IsLifetimeStart ?
942 Intrinsic::lifetime_start : Intrinsic::invariant_start;
943 auto Args = F->getFunctionType()->params();
944 Type* ObjectPtr[1] = {Args[1]};
945 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
946 rename(F);
947 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
948 return true;
949 }
950 }
951
952 bool IsLifetimeEnd = Name.startswith("lifetime.end");
953 if (IsLifetimeEnd || Name.startswith("invariant.end")) {
954 Intrinsic::ID ID = IsLifetimeEnd ?
955 Intrinsic::lifetime_end : Intrinsic::invariant_end;
956
957 auto Args = F->getFunctionType()->params();
958 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
959 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
960 rename(F);
961 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
962 return true;
963 }
964 }
965 if (Name.startswith("invariant.group.barrier")) {
966 // Rename invariant.group.barrier to launder.invariant.group
967 auto Args = F->getFunctionType()->params();
968 Type* ObjectPtr[1] = {Args[0]};
969 rename(F);
970 NewFn = Intrinsic::getDeclaration(F->getParent(),
971 Intrinsic::launder_invariant_group, ObjectPtr);
972 return true;
973
974 }
975
976 break;
977 }
978 case 'm': {
979 if (Name.startswith("masked.load.")) {
980 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
981 if (F->getName() !=
982 Intrinsic::getName(Intrinsic::masked_load, Tys, F->getParent())) {
983 rename(F);
984 NewFn = Intrinsic::getDeclaration(F->getParent(),
985 Intrinsic::masked_load,
986 Tys);
987 return true;
988 }
989 }
990 if (Name.startswith("masked.store.")) {
991 auto Args = F->getFunctionType()->params();
992 Type *Tys[] = { Args[0], Args[1] };
993 if (F->getName() !=
994 Intrinsic::getName(Intrinsic::masked_store, Tys, F->getParent())) {
995 rename(F);
996 NewFn = Intrinsic::getDeclaration(F->getParent(),
997 Intrinsic::masked_store,
998 Tys);
999 return true;
1000 }
1001 }
1002 // Renaming gather/scatter intrinsics with no address space overloading
1003 // to the new overload which includes an address space
1004 if (Name.startswith("masked.gather.")) {
1005 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1006 if (F->getName() !=
1007 Intrinsic::getName(Intrinsic::masked_gather, Tys, F->getParent())) {
1008 rename(F);
1009 NewFn = Intrinsic::getDeclaration(F->getParent(),
1010 Intrinsic::masked_gather, Tys);
1011 return true;
1012 }
1013 }
1014 if (Name.startswith("masked.scatter.")) {
1015 auto Args = F->getFunctionType()->params();
1016 Type *Tys[] = {Args[0], Args[1]};
1017 if (F->getName() !=
1018 Intrinsic::getName(Intrinsic::masked_scatter, Tys, F->getParent())) {
1019 rename(F);
1020 NewFn = Intrinsic::getDeclaration(F->getParent(),
1021 Intrinsic::masked_scatter, Tys);
1022 return true;
1023 }
1024 }
1025 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1026 // alignment parameter to embedding the alignment as an attribute of
1027 // the pointer args.
1028 if (Name.startswith("memcpy.") && F->arg_size() == 5) {
1029 rename(F);
1030 // Get the types of dest, src, and len
1031 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
1032 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
1033 ParamTypes);
1034 return true;
1035 }
1036 if (Name.startswith("memmove.") && F->arg_size() == 5) {
1037 rename(F);
1038 // Get the types of dest, src, and len
1039 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
1040 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
1041 ParamTypes);
1042 return true;
1043 }
1044 if (Name.startswith("memset.") && F->arg_size() == 5) {
1045 rename(F);
1046 // Get the types of dest, and len
1047 const auto *FT = F->getFunctionType();
1048 Type *ParamTypes[2] = {
1049 FT->getParamType(0), // Dest
1050 FT->getParamType(2) // len
1051 };
1052 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
1053 ParamTypes);
1054 return true;
1055 }
1056 break;
1057 }
1058 case 'n': {
1059 if (Name.startswith("nvvm.")) {
1060 Name = Name.substr(5);
1061
1062 // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
1063 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
1064 .Cases("brev32", "brev64", Intrinsic::bitreverse)
1065 .Case("clz.i", Intrinsic::ctlz)
1066 .Case("popc.i", Intrinsic::ctpop)
1067 .Default(Intrinsic::not_intrinsic);
1068 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
1069 NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
1070 {F->getReturnType()});
1071 return true;
1072 }
1073
1074 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1075 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1076 //
1077 // TODO: We could add lohi.i2d.
1078 bool Expand = StringSwitch<bool>(Name)
1079 .Cases("abs.i", "abs.ll", true)
1080 .Cases("clz.ll", "popc.ll", "h2f", true)
1081 .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
1082 .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
1083 .StartsWith("atomic.load.add.f32.p", true)
1084 .StartsWith("atomic.load.add.f64.p", true)
1085 .Default(false);
1086 if (Expand) {
1087 NewFn = nullptr;
1088 return true;
1089 }
1090 }
1091 break;
1092 }
1093 case 'o':
1094 // We only need to change the name to match the mangling including the
1095 // address space.
1096 if (Name.startswith("objectsize.")) {
1097 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1098 if (F->arg_size() == 2 || F->arg_size() == 3 ||
1099 F->getName() !=
1100 Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) {
1101 rename(F);
1102 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
1103 Tys);
1104 return true;
1105 }
1106 }
1107 break;
1108
1109 case 'p':
1110 if (Name == "prefetch") {
1111 // Handle address space overloading.
1112 Type *Tys[] = {F->arg_begin()->getType()};
1113 if (F->getName() !=
1114 Intrinsic::getName(Intrinsic::prefetch, Tys, F->getParent())) {
1115 rename(F);
1116 NewFn =
1117 Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
1118 return true;
1119 }
1120 } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
1121 rename(F);
1122 NewFn = Intrinsic::getDeclaration(
1123 F->getParent(), Intrinsic::ptr_annotation,
1124 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1125 return true;
1126 }
1127 break;
1128
1129 case 's':
1130 if (Name == "stackprotectorcheck") {
1131 NewFn = nullptr;
1132 return true;
1133 }
1134 break;
1135
1136 case 'v': {
1137 if (Name == "var.annotation" && F->arg_size() == 4) {
1138 rename(F);
1139 NewFn = Intrinsic::getDeclaration(
1140 F->getParent(), Intrinsic::var_annotation,
1141 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1142 return true;
1143 }
1144 break;
1145 }
1146
1147 case 'w':
1148 if (Name.startswith("wasm.fma.")) {
1149 rename(F);
1150 NewFn = Intrinsic::getDeclaration(
1151 F->getParent(), Intrinsic::wasm_relaxed_madd, F->getReturnType());
1152 return true;
1153 }
1154 if (Name.startswith("wasm.fms.")) {
1155 rename(F);
1156 NewFn = Intrinsic::getDeclaration(
1157 F->getParent(), Intrinsic::wasm_relaxed_nmadd, F->getReturnType());
1158 return true;
1159 }
1160 if (Name.startswith("wasm.laneselect.")) {
1161 rename(F);
1162 NewFn = Intrinsic::getDeclaration(
1163 F->getParent(), Intrinsic::wasm_relaxed_laneselect,
1164 F->getReturnType());
1165 return true;
1166 }
1167 if (Name == "wasm.dot.i8x16.i7x16.signed") {
1168 rename(F);
1169 NewFn = Intrinsic::getDeclaration(
1170 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed);
1171 return true;
1172 }
1173 if (Name == "wasm.dot.i8x16.i7x16.add.signed") {
1174 rename(F);
1175 NewFn = Intrinsic::getDeclaration(
1176 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
1177 return true;
1178 }
1179 break;
1180
1181 case 'x':
1182 if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
1183 return true;
1184 }
1185
1186 auto *ST = dyn_cast<StructType>(F->getReturnType());
1187 if (ST && (!ST->isLiteral() || ST->isPacked())) {
1188 // Replace return type with literal non-packed struct. Only do this for
1189 // intrinsics declared to return a struct, not for intrinsics with
1190 // overloaded return type, in which case the exact struct type will be
1191 // mangled into the name.
1192 SmallVector<Intrinsic::IITDescriptor> Desc;
1193 Intrinsic::getIntrinsicInfoTableEntries(F->getIntrinsicID(), Desc);
1194 if (Desc.front().Kind == Intrinsic::IITDescriptor::Struct) {
1195 auto *FT = F->getFunctionType();
1196 auto *NewST = StructType::get(ST->getContext(), ST->elements());
1197 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1198 std::string Name = F->getName().str();
1199 rename(F);
1200 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1201 Name, F->getParent());
1202
1203 // The new function may also need remangling.
1204 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
1205 NewFn = *Result;
1206 return true;
1207 }
1208 }
1209
1210 // Remangle our intrinsic since we upgrade the mangling
1211 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1212 if (Result != std::nullopt) {
1213 NewFn = *Result;
1214 return true;
1215 }
1216
1217 // This may not belong here. This function is effectively being overloaded
1218 // to both detect an intrinsic which needs upgrading, and to provide the
1219 // upgraded form of the intrinsic. We should perhaps have two separate
1220 // functions for this.
1221 return false;
1222}
1223
1224bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
1225 NewFn = nullptr;
1226 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
1227 assert(F != NewFn && "Intrinsic function upgraded to the same function");
1228
1229 // Upgrade intrinsic attributes. This does not change the function.
1230 if (NewFn)
1231 F = NewFn;
1232 if (Intrinsic::ID id = F->getIntrinsicID())
1233 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
1234 return Upgraded;
1235}
1236
1237GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
1238 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
1239 GV->getName() == "llvm.global_dtors")) ||
1240 !GV->hasInitializer())
1241 return nullptr;
1242 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1243 if (!ATy)
1244 return nullptr;
1245 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1246 if (!STy || STy->getNumElements() != 2)
1247 return nullptr;
1248
1249 LLVMContext &C = GV->getContext();
1250 IRBuilder<> IRB(C);
1251 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1252 IRB.getInt8PtrTy());
1253 Constant *Init = GV->getInitializer();
1254 unsigned N = Init->getNumOperands();
1255 std::vector<Constant *> NewCtors(N);
1256 for (unsigned i = 0; i != N; ++i) {
1257 auto Ctor = cast<Constant>(Init->getOperand(i));
1258 NewCtors[i] = ConstantStruct::get(
1259 EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1260 Constant::getNullValue(IRB.getInt8PtrTy()));
1261 }
1262 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1263
1264 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1265 NewInit, GV->getName());
1266}
1267
1268// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1269// to byte shuffles.
1270static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
1271 Value *Op, unsigned Shift) {
1272 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1273 unsigned NumElts = ResultTy->getNumElements() * 8;
1274
1275 // Bitcast from a 64-bit element type to a byte element type.
1276 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1277 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1278
1279 // We'll be shuffling in zeroes.
1280 Value *Res = Constant::getNullValue(VecTy);
1281
1282 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1283 // we'll just return the zero vector.
1284 if (Shift < 16) {
1285 int Idxs[64];
1286 // 256/512-bit version is split into 2/4 16-byte lanes.
1287 for (unsigned l = 0; l != NumElts; l += 16)
1288 for (unsigned i = 0; i != 16; ++i) {
1289 unsigned Idx = NumElts + i - Shift;
1290 if (Idx < NumElts)
1291 Idx -= NumElts - 16; // end of lane, switch operand.
1292 Idxs[l + i] = Idx + l;
1293 }
1294
1295 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
1296 }
1297
1298 // Bitcast back to a 64-bit element type.
1299 return Builder.CreateBitCast(Res, ResultTy, "cast");
1300}
1301
1302// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1303// to byte shuffles.
1304static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1305 unsigned Shift) {
1306 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1307 unsigned NumElts = ResultTy->getNumElements() * 8;
1308
1309 // Bitcast from a 64-bit element type to a byte element type.
1310 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1311 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1312
1313 // We'll be shuffling in zeroes.
1314 Value *Res = Constant::getNullValue(VecTy);
1315
1316 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1317 // we'll just return the zero vector.
1318 if (Shift < 16) {
1319 int Idxs[64];
1320 // 256/512-bit version is split into 2/4 16-byte lanes.
1321 for (unsigned l = 0; l != NumElts; l += 16)
1322 for (unsigned i = 0; i != 16; ++i) {
1323 unsigned Idx = i + Shift;
1324 if (Idx >= 16)
1325 Idx += NumElts - 16; // end of lane, switch operand.
1326 Idxs[l + i] = Idx + l;
1327 }
1328
1329 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
1330 }
1331
1332 // Bitcast back to a 64-bit element type.
1333 return Builder.CreateBitCast(Res, ResultTy, "cast");
1334}
1335
1336static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
1337 unsigned NumElts) {
1338 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
1339 llvm::VectorType *MaskTy = FixedVectorType::get(
1340 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1341 Mask = Builder.CreateBitCast(Mask, MaskTy);
1342
1343 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
1344 // i8 and we need to extract down to the right number of elements.
1345 if (NumElts <= 4) {
1346 int Indices[4];
1347 for (unsigned i = 0; i != NumElts; ++i)
1348 Indices[i] = i;
1349 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
1350 "extract");
1351 }
1352
1353 return Mask;
1354}
1355
1356static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
1357 Value *Op0, Value *Op1) {
1358 // If the mask is all ones just emit the first operation.
1359 if (const auto *C = dyn_cast<Constant>(Mask))
1360 if (C->isAllOnesValue())
1361 return Op0;
1362
1363 Mask = getX86MaskVec(Builder, Mask,
1364 cast<FixedVectorType>(Op0->getType())->getNumElements());
1365 return Builder.CreateSelect(Mask, Op0, Op1);
1366}
1367
1368static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1369 Value *Op0, Value *Op1) {
1370 // If the mask is all ones just emit the first operation.
1371 if (const auto *C = dyn_cast<Constant>(Mask))
1372 if (C->isAllOnesValue())
1373 return Op0;
1374
1375 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1376 Mask->getType()->getIntegerBitWidth());
1377 Mask = Builder.CreateBitCast(Mask, MaskTy);
1378 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1379 return Builder.CreateSelect(Mask, Op0, Op1);
1380}
1381
1382// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
1383// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
1384// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
1385static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1386 Value *Op1, Value *Shift,
1387 Value *Passthru, Value *Mask,
1388 bool IsVALIGN) {
1389 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1390
1391 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1392 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
1393 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
1394 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
1395
1396 // Mask the immediate for VALIGN.
1397 if (IsVALIGN)
1398 ShiftVal &= (NumElts - 1);
1399
1400 // If palignr is shifting the pair of vectors more than the size of two
1401 // lanes, emit zero.
1402 if (ShiftVal >= 32)
1403 return llvm::Constant::getNullValue(Op0->getType());
1404
1405 // If palignr is shifting the pair of input vectors more than one lane,
1406 // but less than two lanes, convert to shifting in zeroes.
1407 if (ShiftVal > 16) {
1408 ShiftVal -= 16;
1409 Op1 = Op0;
1410 Op0 = llvm::Constant::getNullValue(Op0->getType());
1411 }
1412
1413 int Indices[64];
1414 // 256-bit palignr operates on 128-bit lanes so we need to handle that
1415 for (unsigned l = 0; l < NumElts; l += 16) {
1416 for (unsigned i = 0; i != 16; ++i) {
1417 unsigned Idx = ShiftVal + i;
1418 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
1419 Idx += NumElts - 16; // End of lane, switch operand.
1420 Indices[l + i] = Idx + l;
1421 }
1422 }
1423
1424 Value *Align = Builder.CreateShuffleVector(
1425 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
1426
1427 return EmitX86Select(Builder, Mask, Align, Passthru);
1428}
1429
1430static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
1431 bool ZeroMask, bool IndexForm) {
1432 Type *Ty = CI.getType();
1433 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
1434 unsigned EltWidth = Ty->getScalarSizeInBits();
1435 bool IsFloat = Ty->isFPOrFPVectorTy();
1436 Intrinsic::ID IID;
1437 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1438 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1439 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1440 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1441 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1442 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1443 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1444 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1445 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1446 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1447 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1448 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1449 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1450 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1451 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1452 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1453 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1454 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1455 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1456 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1457 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1458 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1459 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1460 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1461 else if (VecWidth == 128 && EltWidth == 16)
1462 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1463 else if (VecWidth == 256 && EltWidth == 16)
1464 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1465 else if (VecWidth == 512 && EltWidth == 16)
1466 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1467 else if (VecWidth == 128 && EltWidth == 8)
1468 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1469 else if (VecWidth == 256 && EltWidth == 8)
1470 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1471 else if (VecWidth == 512 && EltWidth == 8)
1472 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1473 else
1474 llvm_unreachable("Unexpected intrinsic");
1475
1476 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1477 CI.getArgOperand(2) };
1478
1479 // If this isn't index form we need to swap operand 0 and 1.
1480 if (!IndexForm)
1481 std::swap(Args[0], Args[1]);
1482
1483 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1484 Args);
1485 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
1486 : Builder.CreateBitCast(CI.getArgOperand(1),
1487 Ty);
1488 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1489}
1490
1491static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
1492 Intrinsic::ID IID) {
1493 Type *Ty = CI.getType();
1494 Value *Op0 = CI.getOperand(0);
1495 Value *Op1 = CI.getOperand(1);
1496 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1497 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1498
1499 if (CI.arg_size() == 4) { // For masked intrinsics.
1500 Value *VecSrc = CI.getOperand(2);
1501 Value *Mask = CI.getOperand(3);
1502 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1503 }
1504 return Res;
1505}
1506
1507static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
1508 bool IsRotateRight) {
1509 Type *Ty = CI.getType();
1510 Value *Src = CI.getArgOperand(0);
1511 Value *Amt = CI.getArgOperand(1);
1512
1513 // Amount may be scalar immediate, in which case create a splat vector.
1514 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1515 // we only care about the lowest log2 bits anyway.
1516 if (Amt->getType() != Ty) {
1517 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1518 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1519 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1520 }
1521
1522 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1523 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1524 Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1525
1526 if (CI.arg_size() == 4) { // For masked intrinsics.
1527 Value *VecSrc = CI.getOperand(2);
1528 Value *Mask = CI.getOperand(3);
1529 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1530 }
1531 return Res;
1532}
1533
1534static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
1535 bool IsSigned) {
1536 Type *Ty = CI.getType();
1537 Value *LHS = CI.getArgOperand(0);
1538 Value *RHS = CI.getArgOperand(1);
1539
1540 CmpInst::Predicate Pred;
1541 switch (Imm) {
1542 case 0x0:
1543 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1544 break;
1545 case 0x1:
1546 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1547 break;
1548 case 0x2:
1549 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1550 break;
1551 case 0x3:
1552 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1553 break;
1554 case 0x4:
1555 Pred = ICmpInst::ICMP_EQ;
1556 break;
1557 case 0x5:
1558 Pred = ICmpInst::ICMP_NE;
1559 break;
1560 case 0x6:
1561 return Constant::getNullValue(Ty); // FALSE
1562 case 0x7:
1563 return Constant::getAllOnesValue(Ty); // TRUE
1564 default:
1565 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
1566 }
1567
1568 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1569 Value *Ext = Builder.CreateSExt(Cmp, Ty);
1570 return Ext;
1571}
1572
1573static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
1574 bool IsShiftRight, bool ZeroMask) {
1575 Type *Ty = CI.getType();
1576 Value *Op0 = CI.getArgOperand(0);
1577 Value *Op1 = CI.getArgOperand(1);
1578 Value *Amt = CI.getArgOperand(2);
1579
1580 if (IsShiftRight)
1581 std::swap(Op0, Op1);
1582
1583 // Amount may be scalar immediate, in which case create a splat vector.
1584 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1585 // we only care about the lowest log2 bits anyway.
1586 if (Amt->getType() != Ty) {
1587 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1588 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1589 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1590 }
1591
1592 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1593 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1594 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1595
1596 unsigned NumArgs = CI.arg_size();
1597 if (NumArgs >= 4) { // For masked intrinsics.
1598 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1599 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
1600 CI.getArgOperand(0);
1601 Value *Mask = CI.getOperand(NumArgs - 1);
1602 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1603 }
1604 return Res;
1605}
1606
1607static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1608 Value *Ptr, Value *Data, Value *Mask,
1609 bool Aligned) {
1610 // Cast the pointer to the right type.
1611 Ptr = Builder.CreateBitCast(Ptr,
1612 llvm::PointerType::getUnqual(Data->getType()));
1613 const Align Alignment =
1614 Aligned
1615 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1616 : Align(1);
1617
1618 // If the mask is all ones just emit a regular store.
1619 if (const auto *C = dyn_cast<Constant>(Mask))
1620 if (C->isAllOnesValue())
1621 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1622
1623 // Convert the mask from an integer type to a vector of i1.
1624 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1625 Mask = getX86MaskVec(Builder, Mask, NumElts);
1626 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1627}
1628
1629static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1630 Value *Ptr, Value *Passthru, Value *Mask,
1631 bool Aligned) {
1632 Type *ValTy = Passthru->getType();
1633 // Cast the pointer to the right type.
1634 Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1635 const Align Alignment =
1636 Aligned
1637 ? Align(
1638 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
1639 8)
1640 : Align(1);
1641
1642 // If the mask is all ones just emit a regular store.
1643 if (const auto *C = dyn_cast<Constant>(Mask))
1644 if (C->isAllOnesValue())
1645 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1646
1647 // Convert the mask from an integer type to a vector of i1.
1648 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1649 Mask = getX86MaskVec(Builder, Mask, NumElts);
1650 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
1651}
1652
1653static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
1654 Type *Ty = CI.getType();
1655 Value *Op0 = CI.getArgOperand(0);
1656 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1657 Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
1658 if (CI.arg_size() == 3)
1659 Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1660 return Res;
1661}
1662
1663static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
1664 Type *Ty = CI.getType();
1665
1666 // Arguments have a vXi32 type so cast to vXi64.
1667 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1668 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1669
1670 if (IsSigned) {
1671 // Shift left then arithmetic shift right.
1672 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1673 LHS = Builder.CreateShl(LHS, ShiftAmt);
1674 LHS = Builder.CreateAShr(LHS, ShiftAmt);
1675 RHS = Builder.CreateShl(RHS, ShiftAmt);
1676 RHS = Builder.CreateAShr(RHS, ShiftAmt);
1677 } else {
1678 // Clear the upper bits.
1679 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1680 LHS = Builder.CreateAnd(LHS, Mask);
1681 RHS = Builder.CreateAnd(RHS, Mask);
1682 }
1683
1684 Value *Res = Builder.CreateMul(LHS, RHS);
1685
1686 if (CI.arg_size() == 4)
1687 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1688
1689 return Res;
1690}
1691
1692// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
1693static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1694 Value *Mask) {
1695 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
1696 if (Mask) {
1697 const auto *C = dyn_cast<Constant>(Mask);
1698 if (!C || !C->isAllOnesValue())
1699 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1700 }
1701
1702 if (NumElts < 8) {
1703 int Indices[8];
1704 for (unsigned i = 0; i != NumElts; ++i)
1705 Indices[i] = i;
1706 for (unsigned i = NumElts; i != 8; ++i)
1707 Indices[i] = NumElts + i % NumElts;
1708 Vec = Builder.CreateShuffleVector(Vec,
1709 Constant::getNullValue(Vec->getType()),
1710 Indices);
1711 }
1712 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1713}
1714
1715static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
1716 unsigned CC, bool Signed) {
1717 Value *Op0 = CI.getArgOperand(0);
1718 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1719
1720 Value *Cmp;
1721 if (CC == 3) {
1722 Cmp = Constant::getNullValue(
1723 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1724 } else if (CC == 7) {
1725 Cmp = Constant::getAllOnesValue(
1726 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1727 } else {
1728 ICmpInst::Predicate Pred;
1729 switch (CC) {
1730 default: llvm_unreachable("Unknown condition code");
1731 case 0: Pred = ICmpInst::ICMP_EQ; break;
1732 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
1733 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
1734 case 4: Pred = ICmpInst::ICMP_NE; break;
1735 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
1736 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
1737 }
1738 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1739 }
1740
1741 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
1742
1743 return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
1744}
1745
1746// Replace a masked intrinsic with an older unmasked intrinsic.
1747static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
1748 Intrinsic::ID IID) {
1749 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1750 Value *Rep = Builder.CreateCall(Intrin,
1751 { CI.getArgOperand(0), CI.getArgOperand(1) });
1752 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1753}
1754
1755static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
1756 Value* A = CI.getArgOperand(0);
1757 Value* B = CI.getArgOperand(1);
1758 Value* Src = CI.getArgOperand(2);
1759 Value* Mask = CI.getArgOperand(3);
1760
1761 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1762 Value* Cmp = Builder.CreateIsNotNull(AndNode);
1763 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1764 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1765 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1766 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1767}
1768
1769
1770static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
1771 Value* Op = CI.getArgOperand(0);
1772 Type* ReturnOp = CI.getType();
1773 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
1774 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1775 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1776}
1777
1778// Replace intrinsic with unmasked version and a select.
1779static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1780 CallBase &CI, Value *&Rep) {
1781 Name = Name.substr(12); // Remove avx512.mask.
1782
1783 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
1784 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
1785 Intrinsic::ID IID;
1786 if (Name.startswith("max.p")) {
1787 if (VecWidth == 128 && EltWidth == 32)
1788 IID = Intrinsic::x86_sse_max_ps;
1789 else if (VecWidth == 128 && EltWidth == 64)
1790 IID = Intrinsic::x86_sse2_max_pd;
1791 else if (VecWidth == 256 && EltWidth == 32)
1792 IID = Intrinsic::x86_avx_max_ps_256;
1793 else if (VecWidth == 256 && EltWidth == 64)
1794 IID = Intrinsic::x86_avx_max_pd_256;
1795 else
1796 llvm_unreachable("Unexpected intrinsic");
1797 } else if (Name.startswith("min.p")) {
1798 if (VecWidth == 128 && EltWidth == 32)
1799 IID = Intrinsic::x86_sse_min_ps;
1800 else if (VecWidth == 128 && EltWidth == 64)
1801 IID = Intrinsic::x86_sse2_min_pd;
1802 else if (VecWidth == 256 && EltWidth == 32)
1803 IID = Intrinsic::x86_avx_min_ps_256;
1804 else if (VecWidth == 256 && EltWidth == 64)
1805 IID = Intrinsic::x86_avx_min_pd_256;
1806 else
1807 llvm_unreachable("Unexpected intrinsic");
1808 } else if (Name.startswith("pshuf.b.")) {
1809 if (VecWidth == 128)
1810 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1811 else if (VecWidth == 256)
1812 IID = Intrinsic::x86_avx2_pshuf_b;
1813 else if (VecWidth == 512)
1814 IID = Intrinsic::x86_avx512_pshuf_b_512;
1815 else
1816 llvm_unreachable("Unexpected intrinsic");
1817 } else if (Name.startswith("pmul.hr.sw.")) {
1818 if (VecWidth == 128)
1819 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1820 else if (VecWidth == 256)
1821 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1822 else if (VecWidth == 512)
1823 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1824 else
1825 llvm_unreachable("Unexpected intrinsic");
1826 } else if (Name.startswith("pmulh.w.")) {
1827 if (VecWidth == 128)
1828 IID = Intrinsic::x86_sse2_pmulh_w;
1829 else if (VecWidth == 256)
1830 IID = Intrinsic::x86_avx2_pmulh_w;
1831 else if (VecWidth == 512)
1832 IID = Intrinsic::x86_avx512_pmulh_w_512;
1833 else
1834 llvm_unreachable("Unexpected intrinsic");
1835 } else if (Name.startswith("pmulhu.w.")) {
1836 if (VecWidth == 128)
1837 IID = Intrinsic::x86_sse2_pmulhu_w;
1838 else if (VecWidth == 256)
1839 IID = Intrinsic::x86_avx2_pmulhu_w;
1840 else if (VecWidth == 512)
1841 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1842 else
1843 llvm_unreachable("Unexpected intrinsic");
1844 } else if (Name.startswith("pmaddw.d.")) {
1845 if (VecWidth == 128)
1846 IID = Intrinsic::x86_sse2_pmadd_wd;
1847 else if (VecWidth == 256)
1848 IID = Intrinsic::x86_avx2_pmadd_wd;
1849 else if (VecWidth == 512)
1850 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1851 else
1852 llvm_unreachable("Unexpected intrinsic");
1853 } else if (Name.startswith("pmaddubs.w.")) {
1854 if (VecWidth == 128)
1855 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1856 else if (VecWidth == 256)
1857 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1858 else if (VecWidth == 512)
1859 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1860 else
1861 llvm_unreachable("Unexpected intrinsic");
1862 } else if (Name.startswith("packsswb.")) {
1863 if (VecWidth == 128)
1864 IID = Intrinsic::x86_sse2_packsswb_128;
1865 else if (VecWidth == 256)
1866 IID = Intrinsic::x86_avx2_packsswb;
1867 else if (VecWidth == 512)
1868 IID = Intrinsic::x86_avx512_packsswb_512;
1869 else
1870 llvm_unreachable("Unexpected intrinsic");
1871 } else if (Name.startswith("packssdw.")) {
1872 if (VecWidth == 128)
1873 IID = Intrinsic::x86_sse2_packssdw_128;
1874 else if (VecWidth == 256)
1875 IID = Intrinsic::x86_avx2_packssdw;
1876 else if (VecWidth == 512)
1877 IID = Intrinsic::x86_avx512_packssdw_512;
1878 else
1879 llvm_unreachable("Unexpected intrinsic");
1880 } else if (Name.startswith("packuswb.")) {
1881 if (VecWidth == 128)
1882 IID = Intrinsic::x86_sse2_packuswb_128;
1883 else if (VecWidth == 256)
1884 IID = Intrinsic::x86_avx2_packuswb;
1885 else if (VecWidth == 512)
1886 IID = Intrinsic::x86_avx512_packuswb_512;
1887 else
1888 llvm_unreachable("Unexpected intrinsic");
1889 } else if (Name.startswith("packusdw.")) {
1890 if (VecWidth == 128)
1891 IID = Intrinsic::x86_sse41_packusdw;
1892 else if (VecWidth == 256)
1893 IID = Intrinsic::x86_avx2_packusdw;
1894 else if (VecWidth == 512)
1895 IID = Intrinsic::x86_avx512_packusdw_512;
1896 else
1897 llvm_unreachable("Unexpected intrinsic");
1898 } else if (Name.startswith("vpermilvar.")) {
1899 if (VecWidth == 128 && EltWidth == 32)
1900 IID = Intrinsic::x86_avx_vpermilvar_ps;
1901 else if (VecWidth == 128 && EltWidth == 64)
1902 IID = Intrinsic::x86_avx_vpermilvar_pd;
1903 else if (VecWidth == 256 && EltWidth == 32)
1904 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1905 else if (VecWidth == 256 && EltWidth == 64)
1906 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1907 else if (VecWidth == 512 && EltWidth == 32)
1908 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1909 else if (VecWidth == 512 && EltWidth == 64)
1910 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1911 else
1912 llvm_unreachable("Unexpected intrinsic");
1913 } else if (Name == "cvtpd2dq.256") {
1914 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1915 } else if (Name == "cvtpd2ps.256") {
1916 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1917 } else if (Name == "cvttpd2dq.256") {
1918 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1919 } else if (Name == "cvttps2dq.128") {
1920 IID = Intrinsic::x86_sse2_cvttps2dq;
1921 } else if (Name == "cvttps2dq.256") {
1922 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1923 } else if (Name.startswith("permvar.")) {
1924 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
1925 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1926 IID = Intrinsic::x86_avx2_permps;
1927 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1928 IID = Intrinsic::x86_avx2_permd;
1929 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1930 IID = Intrinsic::x86_avx512_permvar_df_256;
1931 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1932 IID = Intrinsic::x86_avx512_permvar_di_256;
1933 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1934 IID = Intrinsic::x86_avx512_permvar_sf_512;
1935 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1936 IID = Intrinsic::x86_avx512_permvar_si_512;
1937 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1938 IID = Intrinsic::x86_avx512_permvar_df_512;
1939 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1940 IID = Intrinsic::x86_avx512_permvar_di_512;
1941 else if (VecWidth == 128 && EltWidth == 16)
1942 IID = Intrinsic::x86_avx512_permvar_hi_128;
1943 else if (VecWidth == 256 && EltWidth == 16)
1944 IID = Intrinsic::x86_avx512_permvar_hi_256;
1945 else if (VecWidth == 512 && EltWidth == 16)
1946 IID = Intrinsic::x86_avx512_permvar_hi_512;
1947 else if (VecWidth == 128 && EltWidth == 8)
1948 IID = Intrinsic::x86_avx512_permvar_qi_128;
1949 else if (VecWidth == 256 && EltWidth == 8)
1950 IID = Intrinsic::x86_avx512_permvar_qi_256;
1951 else if (VecWidth == 512 && EltWidth == 8)
1952 IID = Intrinsic::x86_avx512_permvar_qi_512;
1953 else
1954 llvm_unreachable("Unexpected intrinsic");
1955 } else if (Name.startswith("dbpsadbw.")) {
1956 if (VecWidth == 128)
1957 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1958 else if (VecWidth == 256)
1959 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1960 else if (VecWidth == 512)
1961 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1962 else
1963 llvm_unreachable("Unexpected intrinsic");
1964 } else if (Name.startswith("pmultishift.qb.")) {
1965 if (VecWidth == 128)
1966 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1967 else if (VecWidth == 256)
1968 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1969 else if (VecWidth == 512)
1970 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1971 else
1972 llvm_unreachable("Unexpected intrinsic");
1973 } else if (Name.startswith("conflict.")) {
1974 if (Name[9] == 'd' && VecWidth == 128)
1975 IID = Intrinsic::x86_avx512_conflict_d_128;
1976 else if (Name[9] == 'd' && VecWidth == 256)
1977 IID = Intrinsic::x86_avx512_conflict_d_256;
1978 else if (Name[9] == 'd' && VecWidth == 512)
1979 IID = Intrinsic::x86_avx512_conflict_d_512;
1980 else if (Name[9] == 'q' && VecWidth == 128)
1981 IID = Intrinsic::x86_avx512_conflict_q_128;
1982 else if (Name[9] == 'q' && VecWidth == 256)
1983 IID = Intrinsic::x86_avx512_conflict_q_256;
1984 else if (Name[9] == 'q' && VecWidth == 512)
1985 IID = Intrinsic::x86_avx512_conflict_q_512;
1986 else
1987 llvm_unreachable("Unexpected intrinsic");
1988 } else if (Name.startswith("pavg.")) {
1989 if (Name[5] == 'b' && VecWidth == 128)
1990 IID = Intrinsic::x86_sse2_pavg_b;
1991 else if (Name[5] == 'b' && VecWidth == 256)
1992 IID = Intrinsic::x86_avx2_pavg_b;
1993 else if (Name[5] == 'b' && VecWidth == 512)
1994 IID = Intrinsic::x86_avx512_pavg_b_512;
1995 else if (Name[5] == 'w' && VecWidth == 128)
1996 IID = Intrinsic::x86_sse2_pavg_w;
1997 else if (Name[5] == 'w' && VecWidth == 256)
1998 IID = Intrinsic::x86_avx2_pavg_w;
1999 else if (Name[5] == 'w' && VecWidth == 512)
2000 IID = Intrinsic::x86_avx512_pavg_w_512;
2001 else
2002 llvm_unreachable("Unexpected intrinsic");
2003 } else
2004 return false;
2005
2006 SmallVector<Value *, 4> Args(CI.args());
2007 Args.pop_back();
2008 Args.pop_back();
2009 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
2010 Args);
2011 unsigned NumArgs = CI.arg_size();
2012 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2013 CI.getArgOperand(NumArgs - 2));
2014 return true;
2015}
2016
2017/// Upgrade comment in call to inline asm that represents an objc retain release
2018/// marker.
2019void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2020 size_t Pos;
2021 if (AsmStr->find("mov\tfp") == 0 &&
2022 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
2023 (Pos = AsmStr->find("# marker")) != std::string::npos) {
2024 AsmStr->replace(Pos, 1, ";");
2025 }
2026}
2027
2028static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2029 IRBuilder<> &Builder) {
2030 if (Name == "mve.vctp64.old") {
2031 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
2032 // correct type.
2033 Value *VCTP = Builder.CreateCall(
2034 Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64),
2035 CI->getArgOperand(0), CI->getName());
2036 Value *C1 = Builder.CreateCall(
2037 Intrinsic::getDeclaration(
2038 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2039 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2040 VCTP);
2041 return Builder.CreateCall(
2042 Intrinsic::getDeclaration(
2043 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2044 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2045 C1);
2046 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2047 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2048 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2049 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2050 Name ==
2051 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2052 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2053 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2054 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2055 Name ==
2056 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2057 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2058 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
2059 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
2060 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
2061 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
2062 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
2063 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
2064 std::vector<Type *> Tys;
2065 unsigned ID = CI->getIntrinsicID();
2066 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
2067 switch (ID) {
2068 case Intrinsic::arm_mve_mull_int_predicated:
2069 case Intrinsic::arm_mve_vqdmull_predicated:
2070 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2071 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
2072 break;
2073 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2074 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2075 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2076 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
2077 V2I1Ty};
2078 break;
2079 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2080 Tys = {CI->getType(), CI->getOperand(0)->getType(),
2081 CI->getOperand(1)->getType(), V2I1Ty};
2082 break;
2083 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2084 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
2085 CI->getOperand(2)->getType(), V2I1Ty};
2086 break;
2087 case Intrinsic::arm_cde_vcx1q_predicated:
2088 case Intrinsic::arm_cde_vcx1qa_predicated:
2089 case Intrinsic::arm_cde_vcx2q_predicated:
2090 case Intrinsic::arm_cde_vcx2qa_predicated:
2091 case Intrinsic::arm_cde_vcx3q_predicated:
2092 case Intrinsic::arm_cde_vcx3qa_predicated:
2093 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
2094 break;
2095 default:
2096 llvm_unreachable("Unhandled Intrinsic!");
2097 }
2098
2099 std::vector<Value *> Ops;
2100 for (Value *Op : CI->args()) {
2101 Type *Ty = Op->getType();
2102 if (Ty->getScalarSizeInBits() == 1) {
2103 Value *C1 = Builder.CreateCall(
2104 Intrinsic::getDeclaration(
2105 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2106 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2107 Op);
2108 Op = Builder.CreateCall(
2109 Intrinsic::getDeclaration(F->getParent(),
2110 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2111 C1);
2112 }
2113 Ops.push_back(Op);
2114 }
2115
2116 Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
2117 return Builder.CreateCall(Fn, Ops, CI->getName());
2118 }
2119 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
2120}
2121
2122/// Upgrade a call to an old intrinsic. All argument and return casting must be
2123/// provided to seamlessly integrate with existing context.
2124void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
2125 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
2126 // checks the callee's function type matches. It's likely we need to handle
2127 // type changes here.
2128 Function *F = dyn_cast<Function>(CI->getCalledOperand());
2129 if (!F)
2130 return;
2131
2132 LLVMContext &C = CI->getContext();
2133 IRBuilder<> Builder(C);
2134 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
2135
2136 if (!NewFn) {
2137 // Get the Function's name.
2138 StringRef Name = F->getName();
2139
2140 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
2141 Name = Name.substr(5);
2142
2143 bool IsX86 = Name.startswith("x86.");
2144 if (IsX86)
2145 Name = Name.substr(4);
2146 bool IsNVVM = Name.startswith("nvvm.");
2147 if (IsNVVM)
2148 Name = Name.substr(5);
2149 bool IsARM = Name.startswith("arm.");
2150 if (IsARM)
2151 Name = Name.substr(4);
2152
2153 if (IsX86 && Name.startswith("sse4a.movnt.")) {
2154 Module *M = F->getParent();
2155 SmallVector<Metadata *, 1> Elts;
2156 Elts.push_back(
2157 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2158 MDNode *Node = MDNode::get(C, Elts);
2159
2160 Value *Arg0 = CI->getArgOperand(0);
2161 Value *Arg1 = CI->getArgOperand(1);
2162
2163 // Nontemporal (unaligned) store of the 0'th element of the float/double
2164 // vector.
2165 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
2166 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2167 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
2168 Value *Extract =
2169 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
2170
2171 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
2172 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
2173
2174 // Remove intrinsic.
2175 CI->eraseFromParent();
2176 return;
2177 }
2178
2179 if (IsX86 && (Name.startswith("avx.movnt.") ||
2180 Name.startswith("avx512.storent."))) {
2181 Module *M = F->getParent();
2182 SmallVector<Metadata *, 1> Elts;
2183 Elts.push_back(
2184 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
2185 MDNode *Node = MDNode::get(C, Elts);
2186
2187 Value *Arg0 = CI->getArgOperand(0);
2188 Value *Arg1 = CI->getArgOperand(1);
2189
2190 // Convert the type of the pointer to a pointer to the stored type.
2191 Value *BC = Builder.CreateBitCast(Arg0,
2192 PointerType::getUnqual(Arg1->getType()),
2193 "cast");
2194 StoreInst *SI = Builder.CreateAlignedStore(
2195 Arg1, BC,
2196 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
2197 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
2198
2199 // Remove intrinsic.
2200 CI->eraseFromParent();
2201 return;
2202 }
2203
2204 if (IsX86 && Name == "sse2.storel.dq") {
2205 Value *Arg0 = CI->getArgOperand(0);
2206 Value *Arg1 = CI->getArgOperand(1);
2207
2208 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
2209 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
2210 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
2211 Value *BC = Builder.CreateBitCast(Arg0,
2212 PointerType::getUnqual(Elt->getType()),
2213 "cast");
2214 Builder.CreateAlignedStore(Elt, BC, Align(1));
2215
2216 // Remove intrinsic.
2217 CI->eraseFromParent();
2218 return;
2219 }
2220
2221 if (IsX86 && (Name.startswith("sse.storeu.") ||
2222 Name.startswith("sse2.storeu.") ||
2223 Name.startswith("avx.storeu."))) {
2224 Value *Arg0 = CI->getArgOperand(0);
2225 Value *Arg1 = CI->getArgOperand(1);
2226
2227 Arg0 = Builder.CreateBitCast(Arg0,
2228 PointerType::getUnqual(Arg1->getType()),
2229 "cast");
2230 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
2231
2232 // Remove intrinsic.
2233 CI->eraseFromParent();
2234 return;
2235 }
2236
2237 if (IsX86 && Name == "avx512.mask.store.ss") {
2238 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
2239 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2240 Mask, false);
2241
2242 // Remove intrinsic.
2243 CI->eraseFromParent();
2244 return;
2245 }
2246
2247 if (IsX86 && (Name.startswith("avx512.mask.store"))) {
2248 // "avx512.mask.storeu." or "avx512.mask.store."
2249 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
2250 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2251 CI->getArgOperand(2), Aligned);
2252
2253 // Remove intrinsic.
2254 CI->eraseFromParent();
2255 return;
2256 }
2257
2258 Value *Rep;
2259 // Upgrade packed integer vector compare intrinsics to compare instructions.
2260 if (IsX86 && (Name.startswith("sse2.pcmp") ||
2261 Name.startswith("avx2.pcmp"))) {
2262 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
2263 bool CmpEq = Name[9] == 'e';
2264 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2265 CI->getArgOperand(0), CI->getArgOperand(1));
2266 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
2267 } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
2268 Type *ExtTy = Type::getInt32Ty(C);
2269 if (CI->getOperand(0)->getType()->isIntegerTy(8))
2270 ExtTy = Type::getInt64Ty(C);
2271 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
2272 ExtTy->getPrimitiveSizeInBits();
2273 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
2274 Rep = Builder.CreateVectorSplat(NumElts, Rep);
2275 } else if (IsX86 && (Name == "sse.sqrt.ss" ||
2276 Name == "sse2.sqrt.sd")) {
2277 Value *Vec = CI->getArgOperand(0);
2278 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
2279 Function *Intr = Intrinsic::getDeclaration(F->getParent(),
2280 Intrinsic::sqrt, Elt0->getType());
2281 Elt0 = Builder.CreateCall(Intr, Elt0);
2282 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
2283 } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
2284 Name.startswith("sse2.sqrt.p") ||
2285 Name.startswith("sse.sqrt.p"))) {
2286 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2287 Intrinsic::sqrt,
2288 CI->getType()),
2289 {CI->getArgOperand(0)});
2290 } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
2291 if (CI->arg_size() == 4 &&
2292 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2293 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2294 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
2295 : Intrinsic::x86_avx512_sqrt_pd_512;
2296
2297 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
2298 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
2299 IID), Args);
2300 } else {
2301 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2302 Intrinsic::sqrt,
2303 CI->getType()),
2304 {CI->getArgOperand(0)});
2305 }
2306 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2307 CI->getArgOperand(1));
2308 } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
2309 Name.startswith("avx512.ptestnm"))) {
2310 Value *Op0 = CI->getArgOperand(0);
2311 Value *Op1 = CI->getArgOperand(1);
2312 Value *Mask = CI->getArgOperand(2);
2313 Rep = Builder.CreateAnd(Op0, Op1);
2314 llvm::Type *Ty = Op0->getType();
2315 Value *Zero = llvm::Constant::getNullValue(Ty);
2316 ICmpInst::Predicate Pred =
2317 Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2318 Rep = Builder.CreateICmp(Pred, Rep, Zero);
2319 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
2320 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
2321 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
2322 ->getNumElements();
2323 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
2324 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2325 CI->getArgOperand(1));
2326 } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
2327 unsigned NumElts = CI->getType()->getScalarSizeInBits();
2328 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
2329 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
2330 int Indices[64];
2331 for (unsigned i = 0; i != NumElts; ++i)
2332 Indices[i] = i;
2333
2334 // First extract half of each vector. This gives better codegen than
2335 // doing it in a single shuffle.
2336 LHS =
2337 Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
2338 RHS =
2339 Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
2340 // Concat the vectors.
2341 // NOTE: Operands have to be swapped to match intrinsic definition.
2342 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
2343 Rep = Builder.CreateBitCast(Rep, CI->getType());
2344 } else if (IsX86 && Name == "avx512.kand.w") {
2345 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2346 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2347 Rep = Builder.CreateAnd(LHS, RHS);
2348 Rep = Builder.CreateBitCast(Rep, CI->getType());
2349 } else if (IsX86 && Name == "avx512.kandn.w") {
2350 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2351 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2352 LHS = Builder.CreateNot(LHS);
2353 Rep = Builder.CreateAnd(LHS, RHS);
2354 Rep = Builder.CreateBitCast(Rep, CI->getType());
2355 } else if (IsX86 && Name == "avx512.kor.w") {
2356 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2357 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2358 Rep = Builder.CreateOr(LHS, RHS);
2359 Rep = Builder.CreateBitCast(Rep, CI->getType());
2360 } else if (IsX86 && Name == "avx512.kxor.w") {
2361 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2362 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2363 Rep = Builder.CreateXor(LHS, RHS);
2364 Rep = Builder.CreateBitCast(Rep, CI->getType());
2365 } else if (IsX86 && Name == "avx512.kxnor.w") {
2366 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2367 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2368 LHS = Builder.CreateNot(LHS);
2369 Rep = Builder.CreateXor(LHS, RHS);
2370 Rep = Builder.CreateBitCast(Rep, CI->getType());
2371 } else if (IsX86 && Name == "avx512.knot.w") {
2372 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2373 Rep = Builder.CreateNot(Rep);
2374 Rep = Builder.CreateBitCast(Rep, CI->getType());
2375 } else if (IsX86 &&
2376 (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
2377 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2378 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2379 Rep = Builder.CreateOr(LHS, RHS);
2380 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
2381 Value *C;
2382 if (Name[14] == 'c')
2383 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2384 else
2385 C = ConstantInt::getNullValue(Builder.getInt16Ty());
2386 Rep = Builder.CreateICmpEQ(Rep, C);
2387 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
2388 } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2389 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2390 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2391 Name == "sse.div.ss" || Name == "sse2.div.sd")) {
2392 Type *I32Ty = Type::getInt32Ty(C);
2393 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2394 ConstantInt::get(I32Ty, 0));
2395 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2396 ConstantInt::get(I32Ty, 0));
2397 Value *EltOp;
2398 if (Name.contains(".add."))
2399 EltOp = Builder.CreateFAdd(Elt0, Elt1);
2400 else if (Name.contains(".sub."))
2401 EltOp = Builder.CreateFSub(Elt0, Elt1);
2402 else if (Name.contains(".mul."))
2403 EltOp = Builder.CreateFMul(Elt0, Elt1);
2404 else
2405 EltOp = Builder.CreateFDiv(Elt0, Elt1);
2406 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2407 ConstantInt::get(I32Ty, 0));
2408 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
2409 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
2410 bool CmpEq = Name[16] == 'e';
2411 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2412 } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
2413 Type *OpTy = CI->getArgOperand(0)->getType();
2414 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2415 Intrinsic::ID IID;
2416 switch (VecWidth) {
2417 default: llvm_unreachable("Unexpected intrinsic");
2418 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
2419 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
2420 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
2421 }
2422
2423 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2424 { CI->getOperand(0), CI->getArgOperand(1) });
2425 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2426 } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
2427 Type *OpTy = CI->getArgOperand(0)->getType();
2428 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2429 unsigned EltWidth = OpTy->getScalarSizeInBits();
2430 Intrinsic::ID IID;
2431 if (VecWidth == 128 && EltWidth == 32)
2432 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2433 else if (VecWidth == 256 && EltWidth == 32)
2434 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2435 else if (VecWidth == 512 && EltWidth == 32)
2436 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2437 else if (VecWidth == 128 && EltWidth == 64)
2438 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2439 else if (VecWidth == 256 && EltWidth == 64)
2440 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2441 else if (VecWidth == 512 && EltWidth == 64)
2442 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2443 else
2444 llvm_unreachable("Unexpected intrinsic");
2445
2446 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2447 { CI->getOperand(0), CI->getArgOperand(1) });
2448 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2449 } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
2450 SmallVector<Value *, 4> Args(CI->args());
2451 Type *OpTy = Args[0]->getType();
2452 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2453 unsigned EltWidth = OpTy->getScalarSizeInBits();
2454 Intrinsic::ID IID;
2455 if (VecWidth == 128 && EltWidth == 32)
2456 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2457 else if (VecWidth == 256 && EltWidth == 32)
2458 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2459 else if (VecWidth == 512 && EltWidth == 32)
2460 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2461 else if (VecWidth == 128 && EltWidth == 64)
2462 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2463 else if (VecWidth == 256 && EltWidth == 64)
2464 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2465 else if (VecWidth == 512 && EltWidth == 64)
2466 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2467 else
2468 llvm_unreachable("Unexpected intrinsic");
2469
2470 Value *Mask = Constant::getAllOnesValue(CI->getType());
2471 if (VecWidth == 512)
2472 std::swap(Mask, Args.back());
2473 Args.push_back(Mask);
2474
2475 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2476 Args);
2477 } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
2478 // Integer compare intrinsics.
2479 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2480 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2481 } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
2482 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2483 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2484 } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
2485 Name.startswith("avx512.cvtw2mask.") ||
2486 Name.startswith("avx512.cvtd2mask.") ||
2487 Name.startswith("avx512.cvtq2mask."))) {
2488 Value *Op = CI->getArgOperand(0);
2489 Value *Zero = llvm::Constant::getNullValue(Op->getType());
2490 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2491 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2492 } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
2493 Name == "ssse3.pabs.w.128" ||
2494 Name == "ssse3.pabs.d.128" ||
2495 Name.startswith("avx2.pabs") ||
2496 Name.startswith("avx512.mask.pabs"))) {
2497 Rep = upgradeAbs(Builder, *CI);
2498 } else if (IsX86 && (Name == "sse41.pmaxsb" ||
2499 Name == "sse2.pmaxs.w" ||
2500 Name == "sse41.pmaxsd" ||
2501 Name.startswith("avx2.pmaxs") ||
2502 Name.startswith("avx512.mask.pmaxs"))) {
2503 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2504 } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
2505 Name == "sse41.pmaxuw" ||
2506 Name == "sse41.pmaxud" ||
2507 Name.startswith("avx2.pmaxu") ||
2508 Name.startswith("avx512.mask.pmaxu"))) {
2509 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2510 } else if (IsX86 && (Name == "sse41.pminsb" ||
2511 Name == "sse2.pmins.w" ||
2512 Name == "sse41.pminsd" ||
2513 Name.startswith("avx2.pmins") ||
2514 Name.startswith("avx512.mask.pmins"))) {
2515 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2516 } else if (IsX86 && (Name == "sse2.pminu.b" ||
2517 Name == "sse41.pminuw" ||
2518 Name == "sse41.pminud" ||
2519 Name.startswith("avx2.pminu") ||
2520 Name.startswith("avx512.mask.pminu"))) {
2521 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2522 } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
2523 Name == "avx2.pmulu.dq" ||
2524 Name == "avx512.pmulu.dq.512" ||
2525 Name.startswith("avx512.mask.pmulu.dq."))) {
2526 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2527 } else if (IsX86 && (Name == "sse41.pmuldq" ||
2528 Name == "avx2.pmul.dq" ||
2529 Name == "avx512.pmul.dq.512" ||
2530 Name.startswith("avx512.mask.pmul.dq."))) {
2531 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2532 } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
2533 Name == "sse2.cvtsi2sd" ||
2534 Name == "sse.cvtsi642ss" ||
2535 Name == "sse2.cvtsi642sd")) {
2536 Rep = Builder.CreateSIToFP(
2537 CI->getArgOperand(1),
2538 cast<VectorType>(CI->getType())->getElementType());
2539 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2540 } else if (IsX86 && Name == "avx512.cvtusi2sd") {
2541 Rep = Builder.CreateUIToFP(
2542 CI->getArgOperand(1),
2543 cast<VectorType>(CI->getType())->getElementType());
2544 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2545 } else if (IsX86 && Name == "sse2.cvtss2sd") {
2546 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2547 Rep = Builder.CreateFPExt(
2548 Rep, cast<VectorType>(CI->getType())->getElementType());
2549 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2550 } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
2551 Name == "sse2.cvtdq2ps" ||
2552 Name == "avx.cvtdq2.pd.256" ||
2553 Name == "avx.cvtdq2.ps.256" ||
2554 Name.startswith("avx512.mask.cvtdq2pd.") ||
2555 Name.startswith("avx512.mask.cvtudq2pd.") ||
2556 Name.startswith("avx512.mask.cvtdq2ps.") ||
2557 Name.startswith("avx512.mask.cvtudq2ps.") ||
2558 Name.startswith("avx512.mask.cvtqq2pd.") ||
2559 Name.startswith("avx512.mask.cvtuqq2pd.") ||
2560 Name == "avx512.mask.cvtqq2ps.256" ||
2561 Name == "avx512.mask.cvtqq2ps.512" ||
2562 Name == "avx512.mask.cvtuqq2ps.256" ||
2563 Name == "avx512.mask.cvtuqq2ps.512" ||
2564 Name == "sse2.cvtps2pd" ||
2565 Name == "avx.cvt.ps2.pd.256" ||
2566 Name == "avx512.mask.cvtps2pd.128" ||
2567 Name == "avx512.mask.cvtps2pd.256")) {
2568 auto *DstTy = cast<FixedVectorType>(CI->getType());
2569 Rep = CI->getArgOperand(0);
2570 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2571
2572 unsigned NumDstElts = DstTy->getNumElements();
2573 if (NumDstElts < SrcTy->getNumElements()) {
2574 assert(NumDstElts == 2 && "Unexpected vector size");
2575 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2576 }
2577
2578 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2579 bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
2580 if (IsPS2PD)
2581 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2582 else if (CI->arg_size() == 4 &&
2583 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2584 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2585 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2586 : Intrinsic::x86_avx512_sitofp_round;
2587 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2588 { DstTy, SrcTy });
2589 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2590 } else {
2591 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2592 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2593 }
2594
2595 if (CI->arg_size() >= 3)
2596 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2597 CI->getArgOperand(1));
2598 } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
2599 Name.startswith("vcvtph2ps."))) {
2600 auto *DstTy = cast<FixedVectorType>(CI->getType());
2601 Rep = CI->getArgOperand(0);
2602 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2603 unsigned NumDstElts = DstTy->getNumElements();
2604 if (NumDstElts != SrcTy->getNumElements()) {
2605 assert(NumDstElts == 4 && "Unexpected vector size");
2606 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2607 }
2608 Rep = Builder.CreateBitCast(
2609 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
2610 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2611 if (CI->arg_size() >= 3)
2612 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2613 CI->getArgOperand(1));
2614 } else if (IsX86 && Name.startswith("avx512.mask.load")) {
2615 // "avx512.mask.loadu." or "avx512.mask.load."
2616 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
2617 Rep =
2618 UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2619 CI->getArgOperand(2), Aligned);
2620 } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
2621 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2622 Type *PtrTy = ResultTy->getElementType();
2623
2624 // Cast the pointer to element type.
2625 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2626 llvm::PointerType::getUnqual(PtrTy));
2627
2628 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2629 ResultTy->getNumElements());
2630
2631 Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2632 Intrinsic::masked_expandload,
2633 ResultTy);
2634 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2635 } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
2636 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2637 Type *PtrTy = ResultTy->getElementType();
2638
2639 // Cast the pointer to element type.
2640 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2641 llvm::PointerType::getUnqual(PtrTy));
2642
2643 Value *MaskVec =
2644 getX86MaskVec(Builder, CI->getArgOperand(2),
2645 cast<FixedVectorType>(ResultTy)->getNumElements());
2646
2647 Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2648 Intrinsic::masked_compressstore,
2649 ResultTy);
2650 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2651 } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
2652 Name.startswith("avx512.mask.expand."))) {
2653 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2654
2655 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2656 ResultTy->getNumElements());
2657
2658 bool IsCompress = Name[12] == 'c';
2659 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2660 : Intrinsic::x86_avx512_mask_expand;
2661 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2662 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2663 MaskVec });
2664 } else if (IsX86 && Name.startswith("xop.vpcom")) {
2665 bool IsSigned;
2666 if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
2667 Name.endswith("uq"))
2668 IsSigned = false;
2669 else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
2670 Name.endswith("q"))
2671 IsSigned = true;
2672 else
2673 llvm_unreachable("Unknown suffix");
2674
2675 unsigned Imm;
2676 if (CI->arg_size() == 3) {
2677 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2678 } else {
2679 Name = Name.substr(9); // strip off "xop.vpcom"
2680 if (Name.startswith("lt"))
2681 Imm = 0;
2682 else if (Name.startswith("le"))
2683 Imm = 1;
2684 else if (Name.startswith("gt"))
2685 Imm = 2;
2686 else if (Name.startswith("ge"))
2687 Imm = 3;
2688 else if (Name.startswith("eq"))
2689 Imm = 4;
2690 else if (Name.startswith("ne"))
2691 Imm = 5;
2692 else if (Name.startswith("false"))
2693 Imm = 6;
2694 else if (Name.startswith("true"))
2695 Imm = 7;
2696 else
2697 llvm_unreachable("Unknown condition");
2698 }
2699
2700 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2701 } else if (IsX86 && Name.startswith("xop.vpcmov")) {
2702 Value *Sel = CI->getArgOperand(2);
2703 Value *NotSel = Builder.CreateNot(Sel);
2704 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2705 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2706 Rep = Builder.CreateOr(Sel0, Sel1);
2707 } else if (IsX86 && (Name.startswith("xop.vprot") ||
2708 Name.startswith("avx512.prol") ||
2709 Name.startswith("avx512.mask.prol"))) {
2710 Rep = upgradeX86Rotate(Builder, *CI, false);
2711 } else if (IsX86 && (Name.startswith("avx512.pror") ||
2712 Name.startswith("avx512.mask.pror"))) {
2713 Rep = upgradeX86Rotate(Builder, *CI, true);
2714 } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
2715 Name.startswith("avx512.mask.vpshld") ||
2716 Name.startswith("avx512.maskz.vpshld"))) {
2717 bool ZeroMask = Name[11] == 'z';
2718 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2719 } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
2720 Name.startswith("avx512.mask.vpshrd") ||
2721 Name.startswith("avx512.maskz.vpshrd"))) {
2722 bool ZeroMask = Name[11] == 'z';
2723 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2724 } else if (IsX86 && Name == "sse42.crc32.64.8") {
2725 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2726 Intrinsic::x86_sse42_crc32_32_8);
2727 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2728 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2729 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2730 } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
2731 Name.startswith("avx512.vbroadcast.s"))) {
2732 // Replace broadcasts with a series of insertelements.
2733 auto *VecTy = cast<FixedVectorType>(CI->getType());
2734 Type *EltTy = VecTy->getElementType();
2735 unsigned EltNum = VecTy->getNumElements();
2736 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2737 EltTy->getPointerTo());
2738 Value *Load = Builder.CreateLoad(EltTy, Cast);
2739 Type *I32Ty = Type::getInt32Ty(C);
2740 Rep = PoisonValue::get(VecTy);
2741 for (unsigned I = 0; I < EltNum; ++I)
2742 Rep = Builder.CreateInsertElement(Rep, Load,
2743 ConstantInt::get(I32Ty, I));
2744 } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
2745 Name.startswith("sse41.pmovzx") ||
2746 Name.startswith("avx2.pmovsx") ||
2747 Name.startswith("avx2.pmovzx") ||
2748 Name.startswith("avx512.mask.pmovsx") ||
2749 Name.startswith("avx512.mask.pmovzx"))) {
2750 auto *DstTy = cast<FixedVectorType>(CI->getType());
2751 unsigned NumDstElts = DstTy->getNumElements();
2752
2753 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
2754 SmallVector<int, 8> ShuffleMask(NumDstElts);
2755 for (unsigned i = 0; i != NumDstElts; ++i)
2756 ShuffleMask[i] = i;
2757
2758 Value *SV =
2759 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2760
2761 bool DoSext = (StringRef::npos != Name.find("pmovsx"));
2762 Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2763 : Builder.CreateZExt(SV, DstTy);
2764 // If there are 3 arguments, it's a masked intrinsic so we need a select.
2765 if (CI->arg_size() == 3)
2766 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2767 CI->getArgOperand(1));
2768 } else if (Name == "avx512.mask.pmov.qd.256" ||
2769 Name == "avx512.mask.pmov.qd.512" ||
2770 Name == "avx512.mask.pmov.wb.256" ||
2771 Name == "avx512.mask.pmov.wb.512") {
2772 Type *Ty = CI->getArgOperand(1)->getType();
2773 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2774 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2775 CI->getArgOperand(1));
2776 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
2777 Name == "avx2.vbroadcasti128")) {
2778 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
2779 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2780 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
2781 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
2782 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2783 PointerType::getUnqual(VT));
2784 Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2785 if (NumSrcElts == 2)
2786 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
2787 else
2788 Rep = Builder.CreateShuffleVector(
2789 Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
2790 } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
2791 Name.startswith("avx512.mask.shuf.f"))) {
2792 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2793 Type *VT = CI->getType();
2794 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
2795 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
2796 unsigned ControlBitsMask = NumLanes - 1;
2797 unsigned NumControlBits = NumLanes / 2;
2798 SmallVector<int, 8> ShuffleMask(0);
2799
2800 for (unsigned l = 0; l != NumLanes; ++l) {
2801 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2802 // We actually need the other source.
2803 if (l >= NumLanes / 2)
2804 LaneMask += NumLanes;
2805 for (unsigned i = 0; i != NumElementsInLane; ++i)
2806 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2807 }
2808 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2809 CI->getArgOperand(1), ShuffleMask);
2810 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2811 CI->getArgOperand(3));
2812 }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
2813 Name.startswith("avx512.mask.broadcasti"))) {
2814 unsigned NumSrcElts =
2815 cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2816 ->getNumElements();
2817 unsigned NumDstElts =
2818 cast<FixedVectorType>(CI->getType())->getNumElements();
2819
2820 SmallVector<int, 8> ShuffleMask(NumDstElts);
2821 for (unsigned i = 0; i != NumDstElts; ++i)
2822 ShuffleMask[i] = i % NumSrcElts;
2823
2824 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2825 CI->getArgOperand(0),
2826 ShuffleMask);
2827 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2828 CI->getArgOperand(1));
2829 } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
2830 Name.startswith("avx2.vbroadcast") ||
2831 Name.startswith("avx512.pbroadcast") ||
2832 Name.startswith("avx512.mask.broadcast.s"))) {
2833 // Replace vp?broadcasts with a vector shuffle.
2834 Value *Op = CI->getArgOperand(0);
2835 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2836 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
2837 SmallVector<int, 8> M;
2838 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
2839 Rep = Builder.CreateShuffleVector(Op, M);
2840
2841 if (CI->arg_size() == 3)
2842 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2843 CI->getArgOperand(1));
2844 } else if (IsX86 && (Name.startswith("sse2.padds.") ||
2845 Name.startswith("avx2.padds.") ||
2846 Name.startswith("avx512.padds.") ||
2847 Name.startswith("avx512.mask.padds."))) {
2848 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2849 } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
2850 Name.startswith("avx2.psubs.") ||
2851 Name.startswith("avx512.psubs.") ||
2852 Name.startswith("avx512.mask.psubs."))) {
2853 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2854 } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
2855 Name.startswith("avx2.paddus.") ||
2856 Name.startswith("avx512.mask.paddus."))) {
2857 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2858 } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
2859 Name.startswith("avx2.psubus.") ||
2860 Name.startswith("avx512.mask.psubus."))) {
2861 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2862 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
2863 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2864 CI->getArgOperand(1),
2865 CI->getArgOperand(2),
2866 CI->getArgOperand(3),
2867 CI->getArgOperand(4),
2868 false);
2869 } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
2870 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2871 CI->getArgOperand(1),
2872 CI->getArgOperand(2),
2873 CI->getArgOperand(3),
2874 CI->getArgOperand(4),
2875 true);
2876 } else if (IsX86 && (Name == "sse2.psll.dq" ||
2877 Name == "avx2.psll.dq")) {
2878 // 128/256-bit shift left specified in bits.
2879 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2880 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2881 Shift / 8); // Shift is in bits.
2882 } else if (IsX86 && (Name == "sse2.psrl.dq" ||
2883 Name == "avx2.psrl.dq")) {
2884 // 128/256-bit shift right specified in bits.
2885 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2886 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2887 Shift / 8); // Shift is in bits.
2888 } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
2889 Name == "avx2.psll.dq.bs" ||
2890 Name == "avx512.psll.dq.512")) {
2891 // 128/256/512-bit shift left specified in bytes.
2892 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2893 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2894 } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
2895 Name == "avx2.psrl.dq.bs" ||
2896 Name == "avx512.psrl.dq.512")) {
2897 // 128/256/512-bit shift right specified in bytes.
2898 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2899 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2900 } else if (IsX86 && (Name == "sse41.pblendw" ||
2901 Name.startswith("sse41.blendp") ||
2902 Name.startswith("avx.blend.p") ||
2903 Name == "avx2.pblendw" ||
2904 Name.startswith("avx2.pblendd."))) {
2905 Value *Op0 = CI->getArgOperand(0);
2906 Value *Op1 = CI->getArgOperand(1);
2907 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2908 auto *VecTy = cast<FixedVectorType>(CI->getType());
2909 unsigned NumElts = VecTy->getNumElements();
2910
2911 SmallVector<int, 16> Idxs(NumElts);
2912 for (unsigned i = 0; i != NumElts; ++i)
2913 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2914
2915 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2916 } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
2917 Name == "avx2.vinserti128" ||
2918 Name.startswith("avx512.mask.insert"))) {
2919 Value *Op0 = CI->getArgOperand(0);
2920 Value *Op1 = CI->getArgOperand(1);
2921 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2922 unsigned DstNumElts =
2923 cast<FixedVectorType>(CI->getType())->getNumElements();
2924 unsigned SrcNumElts =
2925 cast<FixedVectorType>(Op1->getType())->getNumElements();
2926 unsigned Scale = DstNumElts / SrcNumElts;
2927
2928 // Mask off the high bits of the immediate value; hardware ignores those.
2929 Imm = Imm % Scale;
2930
2931 // Extend the second operand into a vector the size of the destination.
2932 SmallVector<int, 8> Idxs(DstNumElts);
2933 for (unsigned i = 0; i != SrcNumElts; ++i)
2934 Idxs[i] = i;
2935 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
2936 Idxs[i] = SrcNumElts;
2937 Rep = Builder.CreateShuffleVector(Op1, Idxs);
2938
2939 // Insert the second operand into the first operand.
2940
2941 // Note that there is no guarantee that instruction lowering will actually
2942 // produce a vinsertf128 instruction for the created shuffles. In
2943 // particular, the 0 immediate case involves no lane changes, so it can
2944 // be handled as a blend.
2945
2946 // Example of shuffle mask for 32-bit elements:
2947 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
2948 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
2949
2950 // First fill with identify mask.
2951 for (unsigned i = 0; i != DstNumElts; ++i)
2952 Idxs[i] = i;
2953 // Then replace the elements where we need to insert.
2954 for (unsigned i = 0; i != SrcNumElts; ++i)
2955 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2956 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2957
2958 // If the intrinsic has a mask operand, handle that.
2959 if (CI->arg_size() == 5)
2960 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2961 CI->getArgOperand(3));
2962 } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
2963 Name == "avx2.vextracti128" ||
2964 Name.startswith("avx512.mask.vextract"))) {
2965 Value *Op0 = CI->getArgOperand(0);
2966 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2967 unsigned DstNumElts =
2968 cast<FixedVectorType>(CI->getType())->getNumElements();
2969 unsigned SrcNumElts =
2970 cast<FixedVectorType>(Op0->getType())->getNumElements();
2971 unsigned Scale = SrcNumElts / DstNumElts;
2972
2973 // Mask off the high bits of the immediate value; hardware ignores those.
2974 Imm = Imm % Scale;
2975
2976 // Get indexes for the subvector of the input vector.
2977 SmallVector<int, 8> Idxs(DstNumElts);
2978 for (unsigned i = 0; i != DstNumElts; ++i) {
2979 Idxs[i] = i + (Imm * DstNumElts);
2980 }
2981 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2982
2983 // If the intrinsic has a mask operand, handle that.
2984 if (CI->arg_size() == 4)
2985 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2986 CI->getArgOperand(2));
2987 } else if (!IsX86 && Name == "stackprotectorcheck") {
2988 Rep = nullptr;
2989 } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
2990 Name.startswith("avx512.mask.perm.di."))) {
2991 Value *Op0 = CI->getArgOperand(0);
2992 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2993 auto *VecTy = cast<FixedVectorType>(CI->getType());
2994 unsigned NumElts = VecTy->getNumElements();
2995
2996 SmallVector<int, 8> Idxs(NumElts);
2997 for (unsigned i = 0; i != NumElts; ++i)
2998 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
2999
3000 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3001
3002 if (CI->arg_size() == 4)
3003 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3004 CI->getArgOperand(2));
3005 } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
3006 Name == "avx2.vperm2i128")) {
3007 // The immediate permute control byte looks like this:
3008 // [1:0] - select 128 bits from sources for low half of destination
3009 // [2] - ignore
3010 // [3] - zero low half of destination
3011 // [5:4] - select 128 bits from sources for high half of destination
3012 // [6] - ignore
3013 // [7] - zero high half of destination
3014
3015 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3016
3017 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3018 unsigned HalfSize = NumElts / 2;
3019 SmallVector<int, 8> ShuffleMask(NumElts);
3020
3021 // Determine which operand(s) are actually in use for this instruction.
3022 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3023 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3024
3025 // If needed, replace operands based on zero mask.
3026 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3027 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3028
3029 // Permute low half of result.
3030 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3031 for (unsigned i = 0; i < HalfSize; ++i)
3032 ShuffleMask[i] = StartIndex + i;
3033
3034 // Permute high half of result.
3035 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3036 for (unsigned i = 0; i < HalfSize; ++i)
3037 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3038
3039 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3040
3041 } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
3042 Name == "sse2.pshuf.d" ||
3043 Name.startswith("avx512.mask.vpermil.p") ||
3044 Name.startswith("avx512.mask.pshuf.d."))) {
3045 Value *Op0 = CI->getArgOperand(0);
3046 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3047 auto *VecTy = cast<FixedVectorType>(CI->getType());
3048 unsigned NumElts = VecTy->getNumElements();
3049 // Calculate the size of each index in the immediate.
3050 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3051 unsigned IdxMask = ((1 << IdxSize) - 1);
3052
3053 SmallVector<int, 8> Idxs(NumElts);
3054 // Lookup the bits for this element, wrapping around the immediate every
3055 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3056 // to offset by the first index of each group.
3057 for (unsigned i = 0; i != NumElts; ++i)
3058 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3059
3060 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3061
3062 if (CI->arg_size() == 4)
3063 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3064 CI->getArgOperand(2));
3065 } else if (IsX86 && (Name == "sse2.pshufl.w" ||
3066 Name.startswith("avx512.mask.pshufl.w."))) {
3067 Value *Op0 = CI->getArgOperand(0);
3068 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3069 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3070
3071 SmallVector<int, 16> Idxs(NumElts);
3072 for (unsigned l = 0; l != NumElts; l += 8) {
3073 for (unsigned i = 0; i != 4; ++i)
3074 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3075 for (unsigned i = 4; i != 8; ++i)
3076 Idxs[i + l] = i + l;
3077 }
3078
3079 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3080
3081 if (CI->arg_size() == 4)
3082 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3083 CI->getArgOperand(2));
3084 } else if (IsX86 && (Name == "sse2.pshufh.w" ||
3085 Name.startswith("avx512.mask.pshufh.w."))) {
3086 Value *Op0 = CI->getArgOperand(0);
3087 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3088 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3089
3090 SmallVector<int, 16> Idxs(NumElts);
3091 for (unsigned l = 0; l != NumElts; l += 8) {
3092 for (unsigned i = 0; i != 4; ++i)
3093 Idxs[i + l] = i + l;
3094 for (unsigned i = 0; i != 4; ++i)
3095 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3096 }
3097
3098 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3099
3100 if (CI->arg_size() == 4)
3101 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3102 CI->getArgOperand(2));
3103 } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
3104 Value *Op0 = CI->getArgOperand(0);
3105 Value *Op1 = CI->getArgOperand(1);
3106 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3107 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3108
3109 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3110 unsigned HalfLaneElts = NumLaneElts / 2;
3111
3112 SmallVector<int, 16> Idxs(NumElts);
3113 for (unsigned i = 0; i != NumElts; ++i) {
3114 // Base index is the starting element of the lane.
3115 Idxs[i] = i - (i % NumLaneElts);
3116 // If we are half way through the lane switch to the other source.
3117 if ((i % NumLaneElts) >= HalfLaneElts)
3118 Idxs[i] += NumElts;
3119 // Now select the specific element. By adding HalfLaneElts bits from
3120 // the immediate. Wrapping around the immediate every 8-bits.
3121 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3122 }
3123
3124 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3125
3126 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
3127 CI->getArgOperand(3));
3128 } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
3129 Name.startswith("avx512.mask.movshdup") ||
3130 Name.startswith("avx512.mask.movsldup"))) {
3131 Value *Op0 = CI->getArgOperand(0);
3132 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3133 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3134
3135 unsigned Offset = 0;
3136 if (Name.startswith("avx512.mask.movshdup."))
3137 Offset = 1;
3138
3139 SmallVector<int, 16> Idxs(NumElts);
3140 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3141 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3142 Idxs[i + l + 0] = i + l + Offset;
3143 Idxs[i + l + 1] = i + l + Offset;
3144 }
3145
3146 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3147
3148 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
3149 CI->getArgOperand(1));
3150 } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
3151 Name.startswith("avx512.mask.unpckl."))) {
3152 Value *Op0 = CI->getArgOperand(0);
3153 Value *Op1 = CI->getArgOperand(1);
3154 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3155 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3156
3157 SmallVector<int, 64> Idxs(NumElts);
3158 for (int l = 0; l != NumElts; l += NumLaneElts)
3159 for (int i = 0; i != NumLaneElts; ++i)
3160 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3161
3162 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3163
3164 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3165 CI->getArgOperand(2));
3166 } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
3167 Name.startswith("avx512.mask.unpckh."))) {
3168 Value *Op0 = CI->getArgOperand(0);
3169 Value *Op1 = CI->getArgOperand(1);
3170 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3171 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
3172
3173 SmallVector<int, 64> Idxs(NumElts);
3174 for (int l = 0; l != NumElts; l += NumLaneElts)
3175 for (int i = 0; i != NumLaneElts; ++i)
3176 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3177
3178 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3179
3180 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3181 CI->getArgOperand(2));
3182 } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
3183 Name.startswith("avx512.mask.pand."))) {
3184 VectorType *FTy = cast<VectorType>(CI->getType());
3185 VectorType *ITy = VectorType::getInteger(FTy);
3186 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3187 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3188 Rep = Builder.CreateBitCast(Rep, FTy);
3189 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3190 CI->getArgOperand(2));
3191 } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
3192 Name.startswith("avx512.mask.pandn."))) {
3193 VectorType *FTy = cast<VectorType>(CI->getType());
3194 VectorType *ITy = VectorType::getInteger(FTy);
3195 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3196 Rep = Builder.CreateAnd(Rep,
3197 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3198 Rep = Builder.CreateBitCast(Rep, FTy);
3199 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3200 CI->getArgOperand(2));
3201 } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
3202 Name.startswith("avx512.mask.por."))) {
3203 VectorType *FTy = cast<VectorType>(CI->getType());
3204 VectorType *ITy = VectorType::getInteger(FTy);
3205 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3206 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3207 Rep = Builder.CreateBitCast(Rep, FTy);
3208 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3209 CI->getArgOperand(2));
3210 } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
3211 Name.startswith("avx512.mask.pxor."))) {
3212 VectorType *FTy = cast<VectorType>(CI->getType());
3213 VectorType *ITy = VectorType::getInteger(FTy);
3214 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3215 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3216 Rep = Builder.CreateBitCast(Rep, FTy);
3217 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3218 CI->getArgOperand(2));
3219 } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
3220 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3221 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3222 CI->getArgOperand(2));
3223 } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
3224 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3225 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3226 CI->getArgOperand(2));
3227 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
3228 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3229 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3230 CI->getArgOperand(2));
3231 } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
3232 if (Name.endswith(".512")) {
3233 Intrinsic::ID IID;
3234 if (Name[17] == 's')
3235 IID = Intrinsic::x86_avx512_add_ps_512;
3236 else
3237 IID = Intrinsic::x86_avx512_add_pd_512;
3238
3239 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3240 { CI->getArgOperand(0), CI->getArgOperand(1),
3241 CI->getArgOperand(4) });
3242 } else {
3243 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3244 }
3245 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3246 CI->getArgOperand(2));
3247 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
3248 if (Name.endswith(".512")) {
3249 Intrinsic::ID IID;
3250 if (Name[17] == 's')
3251 IID = Intrinsic::x86_avx512_div_ps_512;
3252 else
3253 IID = Intrinsic::x86_avx512_div_pd_512;
3254
3255 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3256 { CI->getArgOperand(0), CI->getArgOperand(1),
3257 CI->getArgOperand(4) });
3258 } else {
3259 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3260 }
3261 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3262 CI->getArgOperand(2));
3263 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
3264 if (Name.endswith(".512")) {
3265 Intrinsic::ID IID;
3266 if (Name[17] == 's')
3267 IID = Intrinsic::x86_avx512_mul_ps_512;
3268 else
3269 IID = Intrinsic::x86_avx512_mul_pd_512;
3270
3271 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3272 { CI->getArgOperand(0), CI->getArgOperand(1),
3273 CI->getArgOperand(4) });
3274 } else {
3275 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
3276 }
3277 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3278 CI->getArgOperand(2));
3279 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
3280 if (Name.endswith(".512")) {
3281 Intrinsic::ID IID;
3282 if (Name[17] == 's')
3283 IID = Intrinsic::x86_avx512_sub_ps_512;
3284 else
3285 IID = Intrinsic::x86_avx512_sub_pd_512;
3286
3287 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3288 { CI->getArgOperand(0), CI->getArgOperand(1),
3289 CI->getArgOperand(4) });
3290 } else {
3291 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
3292 }
3293 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3294 CI->getArgOperand(2));
3295 } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
3296 Name.startswith("avx512.mask.min.p")) &&
3297 Name.drop_front(18) == ".512") {
3298 bool IsDouble = Name[17] == 'd';
3299 bool IsMin = Name[13] == 'i';
3300 static const Intrinsic::ID MinMaxTbl[2][2] = {
3301 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3302 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3303 };
3304 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
3305
3306 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3307 { CI->getArgOperand(0), CI->getArgOperand(1),
3308 CI->getArgOperand(4) });
3309 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
3310 CI->getArgOperand(2));
3311 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
3312 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
3313 Intrinsic::ctlz,
3314 CI->getType()),
3315 { CI->getArgOperand(0), Builder.getInt1(false) });
3316 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
3317 CI->getArgOperand(1));
3318 } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
3319 bool IsImmediate = Name[16] == 'i' ||
3320 (Name.size() > 18 && Name[18] == 'i');
3321 bool IsVariable = Name[16] == 'v';
3322 char Size = Name[16] == '.' ? Name[17] :
3323 Name[17] == '.' ? Name[18] :
3324 Name[18] == '.' ? Name[19] :
3325 Name[20];
3326
3327 Intrinsic::ID IID;
3328 if (IsVariable && Name[17] != '.') {
3329 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
3330 IID = Intrinsic::x86_avx2_psllv_q;
3331 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
3332 IID = Intrinsic::x86_avx2_psllv_q_256;
3333 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
3334 IID = Intrinsic::x86_avx2_psllv_d;
3335 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3336 IID = Intrinsic::x86_avx2_psllv_d_256;
3337 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3338 IID = Intrinsic::x86_avx512_psllv_w_128;
3339 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3340 IID = Intrinsic::x86_avx512_psllv_w_256;
3341 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3342 IID = Intrinsic::x86_avx512_psllv_w_512;
3343 else
3344 llvm_unreachable("Unexpected size");
3345 } else if (Name.endswith(".128")) {
3346 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3347 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3348 : Intrinsic::x86_sse2_psll_d;
3349 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3350 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3351 : Intrinsic::x86_sse2_psll_q;
3352 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3353 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3354 : Intrinsic::x86_sse2_psll_w;
3355 else
3356 llvm_unreachable("Unexpected size");
3357 } else if (Name.endswith(".256")) {
3358 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3359 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3360 : Intrinsic::x86_avx2_psll_d;
3361 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3362 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3363 : Intrinsic::x86_avx2_psll_q;
3364 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3365 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3366 : Intrinsic::x86_avx2_psll_w;
3367 else
3368 llvm_unreachable("Unexpected size");
3369 } else {
3370 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3371 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3372 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3373 Intrinsic::x86_avx512_psll_d_512;
3374 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3375 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3376 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3377 Intrinsic::x86_avx512_psll_q_512;
3378 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3379 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3380 : Intrinsic::x86_avx512_psll_w_512;
3381 else
3382 llvm_unreachable("Unexpected size");
3383 }
3384
3385 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3386 } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
3387 bool IsImmediate = Name[16] == 'i' ||
3388 (Name.size() > 18 && Name[18] == 'i');
3389 bool IsVariable = Name[16] == 'v';
3390 char Size = Name[16] == '.' ? Name[17] :
3391 Name[17] == '.' ? Name[18] :
3392 Name[18] == '.' ? Name[19] :
3393 Name[20];
3394
3395 Intrinsic::ID IID;
3396 if (IsVariable && Name[17] != '.') {
3397 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3398 IID = Intrinsic::x86_avx2_psrlv_q;
3399 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
3400 IID = Intrinsic::x86_avx2_psrlv_q_256;
3401 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
3402 IID = Intrinsic::x86_avx2_psrlv_d;
3403 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
3404 IID = Intrinsic::x86_avx2_psrlv_d_256;
3405 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
3406 IID = Intrinsic::x86_avx512_psrlv_w_128;
3407 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
3408 IID = Intrinsic::x86_avx512_psrlv_w_256;
3409 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
3410 IID = Intrinsic::x86_avx512_psrlv_w_512;
3411 else
3412 llvm_unreachable("Unexpected size");
3413 } else if (Name.endswith(".128")) {
3414 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
3415 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3416 : Intrinsic::x86_sse2_psrl_d;
3417 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
3418 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3419 : Intrinsic::x86_sse2_psrl_q;
3420 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
3421 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3422 : Intrinsic::x86_sse2_psrl_w;
3423 else
3424 llvm_unreachable("Unexpected size");
3425 } else if (Name.endswith(".256")) {
3426 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
3427 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3428 : Intrinsic::x86_avx2_psrl_d;
3429 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
3430 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3431 : Intrinsic::x86_avx2_psrl_q;
3432 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
3433 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3434 : Intrinsic::x86_avx2_psrl_w;
3435 else
3436 llvm_unreachable("Unexpected size");
3437 } else {
3438 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
3439 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3440 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3441 Intrinsic::x86_avx512_psrl_d_512;
3442 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
3443 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3444 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3445 Intrinsic::x86_avx512_psrl_q_512;
3446 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
3447 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3448 : Intrinsic::x86_avx512_psrl_w_512;
3449 else
3450 llvm_unreachable("Unexpected size");
3451 }
3452
3453 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3454 } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
3455 bool IsImmediate = Name[16] == 'i' ||
3456 (Name.size() > 18 && Name[18] == 'i');
3457 bool IsVariable = Name[16] == 'v';
3458 char Size = Name[16] == '.' ? Name[17] :
3459 Name[17] == '.' ? Name[18] :
3460 Name[18] == '.' ? Name[19] :
3461 Name[20];
3462
3463 Intrinsic::ID IID;
3464 if (IsVariable && Name[17] != '.') {
3465 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
3466 IID = Intrinsic::x86_avx2_psrav_d;
3467 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
3468 IID = Intrinsic::x86_avx2_psrav_d_256;
3469 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
3470 IID = Intrinsic::x86_avx512_psrav_w_128;
3471 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
3472 IID = Intrinsic::x86_avx512_psrav_w_256;
3473 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
3474 IID = Intrinsic::x86_avx512_psrav_w_512;
3475 else
3476 llvm_unreachable("Unexpected size");
3477 } else if (Name.endswith(".128")) {
3478 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
3479 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3480 : Intrinsic::x86_sse2_psra_d;
3481 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
3482 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3483 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3484 Intrinsic::x86_avx512_psra_q_128;
3485 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
3486 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3487 : Intrinsic::x86_sse2_psra_w;
3488 else
3489 llvm_unreachable("Unexpected size");
3490 } else if (Name.endswith(".256")) {
3491 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
3492 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3493 : Intrinsic::x86_avx2_psra_d;
3494 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
3495 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3496 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3497 Intrinsic::x86_avx512_psra_q_256;
3498 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
3499 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3500 : Intrinsic::x86_avx2_psra_w;
3501 else
3502 llvm_unreachable("Unexpected size");
3503 } else {
3504 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
3505 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3506 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3507 Intrinsic::x86_avx512_psra_d_512;
3508 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
3509 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3510 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3511 Intrinsic::x86_avx512_psra_q_512;
3512 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
3513 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3514 : Intrinsic::x86_avx512_psra_w_512;
3515 else
3516 llvm_unreachable("Unexpected size");
3517 }
3518
3519 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3520 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
3521 Rep = upgradeMaskedMove(Builder, *CI);
3522 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
3523 Rep = UpgradeMaskToInt(Builder, *CI);
3524 } else if (IsX86 && Name.endswith(".movntdqa")) {
3525 Module *M = F->getParent();
3526 MDNode *Node = MDNode::get(
3527 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3528
3529 Value *Ptr = CI->getArgOperand(0);
3530
3531 // Convert the type of the pointer to a pointer to the stored type.
3532 Value *BC = Builder.CreateBitCast(
3533 Ptr, PointerType::getUnqual(CI->getType()), "cast");
3534 LoadInst *LI = Builder.CreateAlignedLoad(
3535 CI->getType(), BC,
3536 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3537 LI->setMetadata(M->getMDKindID("nontemporal"), Node);
3538 Rep = LI;
3539 } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
3540 Name.startswith("fma.vfmsub.") ||
3541 Name.startswith("fma.vfnmadd.") ||
3542 Name.startswith("fma.vfnmsub."))) {
3543 bool NegMul = Name[6] == 'n';
3544 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3545 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3546
3547 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3548 CI->getArgOperand(2) };
3549
3550 if (IsScalar) {
3551 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3552 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3553 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3554 }
3555
3556 if (NegMul && !IsScalar)
3557 Ops[0] = Builder.CreateFNeg(Ops[0]);
3558 if (NegMul && IsScalar)
3559 Ops[1] = Builder.CreateFNeg(Ops[1]);
3560 if (NegAcc)
3561 Ops[2] = Builder.CreateFNeg(Ops[2]);
3562
3563 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3564 Intrinsic::fma,
3565 Ops[0]->getType()),
3566 Ops);
3567
3568 if (IsScalar)
3569 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3570 (uint64_t)0);
3571 } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
3572 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3573 CI->getArgOperand(2) };
3574
3575 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3576 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3577 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3578
3579 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3580 Intrinsic::fma,
3581 Ops[0]->getType()),
3582 Ops);
3583
3584 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3585 Rep, (uint64_t)0);
3586 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
3587 Name.startswith("avx512.maskz.vfmadd.s") ||
3588 Name.startswith("avx512.mask3.vfmadd.s") ||
3589 Name.startswith("avx512.mask3.vfmsub.s") ||
3590 Name.startswith("avx512.mask3.vfnmsub.s"))) {
3591 bool IsMask3 = Name[11] == '3';
3592 bool IsMaskZ = Name[11] == 'z';
3593 // Drop the "avx512.mask." to make it easier.
3594 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3595 bool NegMul = Name[2] == 'n';
3596 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3597
3598 Value *A = CI->getArgOperand(0);
3599 Value *B = CI->getArgOperand(1);
3600 Value *C = CI->getArgOperand(2);
3601
3602 if (NegMul && (IsMask3 || IsMaskZ))
3603 A = Builder.CreateFNeg(A);
3604 if (NegMul && !(IsMask3 || IsMaskZ))
3605 B = Builder.CreateFNeg(B);
3606 if (NegAcc)
3607 C = Builder.CreateFNeg(C);
3608
3609 A = Builder.CreateExtractElement(A, (uint64_t)0);
3610 B = Builder.CreateExtractElement(B, (uint64_t)0);
3611 C = Builder.CreateExtractElement(C, (uint64_t)0);
3612
3613 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3614 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3615 Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3616
3617 Intrinsic::ID IID;
3618 if (Name.back() == 'd')
3619 IID = Intrinsic::x86_avx512_vfmadd_f64;
3620 else
3621 IID = Intrinsic::x86_avx512_vfmadd_f32;
3622 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3623 Rep = Builder.CreateCall(FMA, Ops);
3624 } else {
3625 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3626 Intrinsic::fma,
3627 A->getType());
3628 Rep = Builder.CreateCall(FMA, { A, B, C });
3629 }
3630
3631 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
3632 IsMask3 ? C : A;
3633
3634 // For Mask3 with NegAcc, we need to create a new extractelement that
3635 // avoids the negation above.
3636 if (NegAcc && IsMask3)
3637 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3638 (uint64_t)0);
3639
3640 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3641 Rep, PassThru);
3642 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3643 Rep, (uint64_t)0);
3644 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
3645 Name.startswith("avx512.mask.vfnmadd.p") ||
3646 Name.startswith("avx512.mask.vfnmsub.p") ||
3647 Name.startswith("avx512.mask3.vfmadd.p") ||
3648 Name.startswith("avx512.mask3.vfmsub.p") ||
3649 Name.startswith("avx512.mask3.vfnmsub.p") ||
3650 Name.startswith("avx512.maskz.vfmadd.p"))) {
3651 bool IsMask3 = Name[11] == '3';
3652 bool IsMaskZ = Name[11] == 'z';
3653 // Drop the "avx512.mask." to make it easier.
3654 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3655 bool NegMul = Name[2] == 'n';
3656 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3657
3658 Value *A = CI->getArgOperand(0);
3659 Value *B = CI->getArgOperand(1);
3660 Value *C = CI->getArgOperand(2);
3661
3662 if (NegMul && (IsMask3 || IsMaskZ))
3663 A = Builder.CreateFNeg(A);
3664 if (NegMul && !(IsMask3 || IsMaskZ))
3665 B = Builder.CreateFNeg(B);
3666 if (NegAcc)
3667 C = Builder.CreateFNeg(C);
3668
3669 if (CI->arg_size() == 5 &&
3670 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3671 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3672 Intrinsic::ID IID;
3673 // Check the character before ".512" in string.
3674 if (Name[Name.size()-5] == 's')
3675 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3676 else
3677 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3678
3679 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3680 { A, B, C, CI->getArgOperand(4) });
3681 } else {
3682 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3683 Intrinsic::fma,
3684 A->getType());
3685 Rep = Builder.CreateCall(FMA, { A, B, C });
3686 }
3687
3688 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3689 IsMask3 ? CI->getArgOperand(2) :
3690 CI->getArgOperand(0);
3691
3692 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3693 } else if (IsX86 && Name.startswith("fma.vfmsubadd.p")) {
3694 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3695 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3696 Intrinsic::ID IID;
3697 if (VecWidth == 128 && EltWidth == 32)
3698 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3699 else if (VecWidth == 256 && EltWidth == 32)
3700 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3701 else if (VecWidth == 128 && EltWidth == 64)
3702 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3703 else if (VecWidth == 256 && EltWidth == 64)
3704 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3705 else
3706 llvm_unreachable("Unexpected intrinsic");
3707
3708 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3709 CI->getArgOperand(2) };
3710 Ops[2] = Builder.CreateFNeg(Ops[2]);
3711 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3712 Ops);
3713 } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
3714 Name.startswith("avx512.mask3.vfmaddsub.p") ||
3715 Name.startswith("avx512.maskz.vfmaddsub.p") ||
3716 Name.startswith("avx512.mask3.vfmsubadd.p"))) {
3717 bool IsMask3 = Name[11] == '3';
3718 bool IsMaskZ = Name[11] == 'z';
3719 // Drop the "avx512.mask." to make it easier.
3720 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3721 bool IsSubAdd = Name[3] == 's';
3722 if (CI->arg_size() == 5) {
3723 Intrinsic::ID IID;
3724 // Check the character before ".512" in string.
3725 if (Name[Name.size()-5] == 's')
3726 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3727 else
3728 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3729
3730 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3731 CI->getArgOperand(2), CI->getArgOperand(4) };
3732 if (IsSubAdd)
3733 Ops[2] = Builder.CreateFNeg(Ops[2]);
3734
3735 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3736 Ops);
3737 } else {
3738 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3739
3740 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3741 CI->getArgOperand(2) };
3742
3743 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3744 Ops[0]->getType());
3745 Value *Odd = Builder.CreateCall(FMA, Ops);
3746 Ops[2] = Builder.CreateFNeg(Ops[2]);
3747 Value *Even = Builder.CreateCall(FMA, Ops);
3748
3749 if (IsSubAdd)
3750 std::swap(Even, Odd);
3751
3752 SmallVector<int, 32> Idxs(NumElts);
3753 for (int i = 0; i != NumElts; ++i)
3754 Idxs[i] = i + (i % 2) * NumElts;
3755
3756 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3757 }
3758
3759 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3760 IsMask3 ? CI->getArgOperand(2) :
3761 CI->getArgOperand(0);
3762
3763 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3764 } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
3765 Name.startswith("avx512.maskz.pternlog."))) {
3766 bool ZeroMask = Name[11] == 'z';
3767 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3768 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3769 Intrinsic::ID IID;
3770 if (VecWidth == 128 && EltWidth == 32)
3771 IID = Intrinsic::x86_avx512_pternlog_d_128;
3772 else if (VecWidth == 256 && EltWidth == 32)
3773 IID = Intrinsic::x86_avx512_pternlog_d_256;
3774 else if (VecWidth == 512 && EltWidth == 32)
3775 IID = Intrinsic::x86_avx512_pternlog_d_512;
3776 else if (VecWidth == 128 && EltWidth == 64)
3777 IID = Intrinsic::x86_avx512_pternlog_q_128;
3778 else if (VecWidth == 256 && EltWidth == 64)
3779 IID = Intrinsic::x86_avx512_pternlog_q_256;
3780 else if (VecWidth == 512 && EltWidth == 64)
3781 IID = Intrinsic::x86_avx512_pternlog_q_512;
3782 else
3783 llvm_unreachable("Unexpected intrinsic");
3784
3785 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3786 CI->getArgOperand(2), CI->getArgOperand(3) };
3787 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3788 Args);
3789 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3790 : CI->getArgOperand(0);
3791 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3792 } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
3793 Name.startswith("avx512.maskz.vpmadd52"))) {
3794 bool ZeroMask = Name[11] == 'z';
3795 bool High = Name[20] == 'h' || Name[21] == 'h';
3796 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3797 Intrinsic::ID IID;
3798 if (VecWidth == 128 && !High)
3799 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3800 else if (VecWidth == 256 && !High)
3801 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3802 else if (VecWidth == 512 && !High)
3803 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3804 else if (VecWidth == 128 && High)
3805 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3806 else if (VecWidth == 256 && High)
3807 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3808 else if (VecWidth == 512 && High)
3809 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3810 else
3811 llvm_unreachable("Unexpected intrinsic");
3812
3813 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3814 CI->getArgOperand(2) };
3815 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3816 Args);
3817 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3818 : CI->getArgOperand(0);
3819 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3820 } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
3821 Name.startswith("avx512.mask.vpermt2var.") ||
3822 Name.startswith("avx512.maskz.vpermt2var."))) {
3823 bool ZeroMask = Name[11] == 'z';
3824 bool IndexForm = Name[17] == 'i';
3825 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3826 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
3827 Name.startswith("avx512.maskz.vpdpbusd.") ||
3828 Name.startswith("avx512.mask.vpdpbusds.") ||
3829 Name.startswith("avx512.maskz.vpdpbusds."))) {
3830 bool ZeroMask = Name[11] == 'z';
3831 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3832 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3833 Intrinsic::ID IID;
3834 if (VecWidth == 128 && !IsSaturating)
3835 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3836 else if (VecWidth == 256 && !IsSaturating)
3837 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3838 else if (VecWidth == 512 && !IsSaturating)
3839 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3840 else if (VecWidth == 128 && IsSaturating)
3841 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3842 else if (VecWidth == 256 && IsSaturating)
3843 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3844 else if (VecWidth == 512 && IsSaturating)
3845 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3846 else
3847 llvm_unreachable("Unexpected intrinsic");
3848
3849 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3850 CI->getArgOperand(2) };
3851 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3852 Args);
3853 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3854 : CI->getArgOperand(0);
3855 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3856 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
3857 Name.startswith("avx512.maskz.vpdpwssd.") ||
3858 Name.startswith("avx512.mask.vpdpwssds.") ||
3859 Name.startswith("avx512.maskz.vpdpwssds."))) {
3860 bool ZeroMask = Name[11] == 'z';
3861 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3862 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3863 Intrinsic::ID IID;
3864 if (VecWidth == 128 && !IsSaturating)
3865 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3866 else if (VecWidth == 256 && !IsSaturating)
3867 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3868 else if (VecWidth == 512 && !IsSaturating)
3869 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3870 else if (VecWidth == 128 && IsSaturating)
3871 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3872 else if (VecWidth == 256 && IsSaturating)
3873 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3874 else if (VecWidth == 512 && IsSaturating)
3875 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3876 else
3877 llvm_unreachable("Unexpected intrinsic");
3878
3879 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3880 CI->getArgOperand(2) };
3881 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3882 Args);
3883 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3884 : CI->getArgOperand(0);
3885 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3886 } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
3887 Name == "addcarry.u32" || Name == "addcarry.u64" ||
3888 Name == "subborrow.u32" || Name == "subborrow.u64")) {
3889 Intrinsic::ID IID;
3890 if (Name[0] == 'a' && Name.back() == '2')
3891 IID = Intrinsic::x86_addcarry_32;
3892 else if (Name[0] == 'a' && Name.back() == '4')
3893 IID = Intrinsic::x86_addcarry_64;
3894 else if (Name[0] == 's' && Name.back() == '2')
3895 IID = Intrinsic::x86_subborrow_32;
3896 else if (Name[0] == 's' && Name.back() == '4')
3897 IID = Intrinsic::x86_subborrow_64;
3898 else
3899 llvm_unreachable("Unexpected intrinsic");
3900
3901 // Make a call with 3 operands.
3902 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3903 CI->getArgOperand(2)};
3904 Value *NewCall = Builder.CreateCall(
3905 Intrinsic::getDeclaration(CI->getModule(), IID),
3906 Args);
3907
3908 // Extract the second result and store it.
3909 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3910 // Cast the pointer to the right type.
3911 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3912 llvm::PointerType::getUnqual(Data->getType()));
3913 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3914 // Replace the original call result with the first result of the new call.
3915 Value *CF = Builder.CreateExtractValue(NewCall, 0);
3916
3917 CI->replaceAllUsesWith(CF);
3918 Rep = nullptr;
3919 } else if (IsX86 && Name.startswith("avx512.mask.") &&
3920 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3921 // Rep will be updated by the call in the condition.
3922 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
3923 Value *Arg = CI->getArgOperand(0);
3924 Value *Neg = Builder.CreateNeg(Arg, "neg");
3925 Value *Cmp = Builder.CreateICmpSGE(
3926 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
3927 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3928 } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
3929 Name.startswith("atomic.load.add.f64.p"))) {
3930 Value *Ptr = CI->getArgOperand(0);
3931 Value *Val = CI->getArgOperand(1);
3932 Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
3933 AtomicOrdering::SequentiallyConsistent);
3934 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
3935 Name == "max.ui" || Name == "max.ull")) {
3936 Value *Arg0 = CI->getArgOperand(0);
3937 Value *Arg1 = CI->getArgOperand(1);
3938 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3939 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3940 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3941 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3942 } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
3943 Name == "min.ui" || Name == "min.ull")) {
3944 Value *Arg0 = CI->getArgOperand(0);
3945 Value *Arg1 = CI->getArgOperand(1);
3946 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3947 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3948 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3949 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3950 } else if (IsNVVM && Name == "clz.ll") {
3951 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
3952 Value *Arg = CI->getArgOperand(0);
3953 Value *Ctlz = Builder.CreateCall(
3954 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3955 {Arg->getType()}),
3956 {Arg, Builder.getFalse()}, "ctlz");
3957 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3958 } else if (IsNVVM && Name == "popc.ll") {
3959 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
3960 // i64.
3961 Value *Arg = CI->getArgOperand(0);
3962 Value *Popc = Builder.CreateCall(
3963 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3964 {Arg->getType()}),
3965 Arg, "ctpop");
3966 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3967 } else if (IsNVVM && Name == "h2f") {
3968 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3969 F->getParent(), Intrinsic::convert_from_fp16,
3970 {Builder.getFloatTy()}),
3971 CI->getArgOperand(0), "h2f");
3972 } else if (IsARM) {
3973 Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder);
3974 } else {
3975 llvm_unreachable("Unknown function for CallBase upgrade.");
3976 }
3977
3978 if (Rep)
3979 CI->replaceAllUsesWith(Rep);
3980 CI->eraseFromParent();
3981 return;
3982 }
3983
3984 const auto &DefaultCase = [&]() -> void {
3985 if (CI->getFunctionType() == NewFn->getFunctionType()) {
3986 // Handle generic mangling change.
3987 assert(
3988 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
3989 "Unknown function for CallBase upgrade and isn't just a name change");
3990 CI->setCalledFunction(NewFn);
3991 return;
3992 }
3993
3994 // This must be an upgrade from a named to a literal struct.
3995 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
3996 assert(OldST != NewFn->getReturnType() &&
3997 "Return type must have changed");
3998 assert(OldST->getNumElements() ==
3999 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
4000 "Must have same number of elements");
4001
4002 SmallVector<Value *> Args(CI->args());
4003 Value *NewCI = Builder.CreateCall(NewFn, Args);
4004 Value *Res = PoisonValue::get(OldST);
4005 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
4006 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
4007 Res = Builder.CreateInsertValue(Res, Elem, Idx);
4008 }
4009 CI->replaceAllUsesWith(Res);
4010 CI->eraseFromParent();
4011 return;
4012 }
4013
4014 // We're probably about to produce something invalid. Let the verifier catch
4015 // it instead of dying here.
4016 CI->setCalledOperand(
4017 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
4018 return;
4019 };
4020 CallInst *NewCall = nullptr;
4021 switch (NewFn->getIntrinsicID()) {
4022 default: {
4023 DefaultCase();
4024 return;
4025 }
4026 case Intrinsic::arm_neon_vst1:
4027 case Intrinsic::arm_neon_vst2:
4028 case Intrinsic::arm_neon_vst3:
4029 case Intrinsic::arm_neon_vst4:
4030 case Intrinsic::arm_neon_vst2lane:
4031 case Intrinsic::arm_neon_vst3lane:
4032 case Intrinsic::arm_neon_vst4lane: {
4033 SmallVector<Value *, 4> Args(CI->args());
4034 NewCall = Builder.CreateCall(NewFn, Args);
4035 break;
4036 }
4037 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4038 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4039 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4040 LLVMContext &Ctx = F->getParent()->getContext();
4041 SmallVector<Value *, 4> Args(CI->args());
4042 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
4043 cast<ConstantInt>(Args[3])->getZExtValue());
4044 NewCall = Builder.CreateCall(NewFn, Args);
4045 break;
4046 }
4047 case Intrinsic::aarch64_sve_ld3_sret:
4048 case Intrinsic::aarch64_sve_ld4_sret:
4049 case Intrinsic::aarch64_sve_ld2_sret: {
4050 StringRef Name = F->getName();
4051 Name = Name.substr(5);
4052 unsigned N = StringSwitch<unsigned>(Name)
4053 .StartsWith("aarch64.sve.ld2", 2)
4054 .StartsWith("aarch64.sve.ld3", 3)
4055 .StartsWith("aarch64.sve.ld4", 4)
4056 .Default(0);
4057 ScalableVectorType *RetTy =
4058 dyn_cast<ScalableVectorType>(F->getReturnType());
4059 unsigned MinElts = RetTy->getMinNumElements() / N;
4060 SmallVector<Value *, 2> Args(CI->args());
4061 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
4062 Value *Ret = llvm::PoisonValue::get(RetTy);
4063 for (unsigned I = 0; I < N; I++) {
4064 Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
4065 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
4066 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
4067 }
4068 NewCall = dyn_cast<CallInst>(Ret);
4069 break;
4070 }
4071
4072 case Intrinsic::vector_extract: {
4073 StringRef Name = F->getName();
4074 Name = Name.substr(5); // Strip llvm
4075 if (!Name.startswith("aarch64.sve.tuple.get")) {
4076 DefaultCase();
4077 return;
4078 }
4079 ScalableVectorType *RetTy =
4080 dyn_cast<ScalableVectorType>(F->getReturnType());
4081 unsigned MinElts = RetTy->getMinNumElements();
4082 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4083 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
4084 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
4085 break;
4086 }
4087
4088 case Intrinsic::vector_insert: {
4089 StringRef Name = F->getName();
4090 Name = Name.substr(5);
4091 if (!Name.startswith("aarch64.sve.tuple")) {
4092 DefaultCase();
4093 return;
4094 }
4095 if (Name.startswith("aarch64.sve.tuple.set")) {
4096 unsigned I = dyn_cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4097 ScalableVectorType *Ty =
4098 dyn_cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
4099 Value *NewIdx =
4100 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
4101 NewCall = Builder.CreateCall(
4102 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
4103 break;
4104 }
4105 if (Name.startswith("aarch64.sve.tuple.create")) {
4106 unsigned N = StringSwitch<unsigned>(Name)
4107 .StartsWith("aa