28#include "llvm/IR/IntrinsicsAArch64.h"
29#include "llvm/IR/IntrinsicsARM.h"
30#include "llvm/IR/IntrinsicsWebAssembly.h"
31#include "llvm/IR/IntrinsicsX86.h"
46 cl::desc(
"Disable autoupgrade of debug info"));
56 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
71 Type *LastArgType =
F->getFunctionType()->getParamType(
72 F->getFunctionType()->getNumParams() - 1);
87 if (
F->getReturnType()->isVectorTy())
97 if (
F->getReturnType()->getScalarType()->isBFloatTy())
107 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
121 if (
Name ==
"addcarryx.u32" ||
122 Name ==
"addcarryx.u64" ||
123 Name ==
"addcarry.u32" ||
124 Name ==
"addcarry.u64" ||
125 Name ==
"subborrow.u32" ||
126 Name ==
"subborrow.u64" ||
127 Name.startswith(
"sse2.padds.") ||
128 Name.startswith(
"sse2.psubs.") ||
129 Name.startswith(
"sse2.paddus.") ||
130 Name.startswith(
"sse2.psubus.") ||
131 Name.startswith(
"avx2.padds.") ||
132 Name.startswith(
"avx2.psubs.") ||
133 Name.startswith(
"avx2.paddus.") ||
134 Name.startswith(
"avx2.psubus.") ||
135 Name.startswith(
"avx512.padds.") ||
136 Name.startswith(
"avx512.psubs.") ||
137 Name.startswith(
"avx512.mask.padds.") ||
138 Name.startswith(
"avx512.mask.psubs.") ||
139 Name.startswith(
"avx512.mask.paddus.") ||
140 Name.startswith(
"avx512.mask.psubus.") ||
141 Name==
"ssse3.pabs.b.128" ||
142 Name==
"ssse3.pabs.w.128" ||
143 Name==
"ssse3.pabs.d.128" ||
144 Name.startswith(
"fma4.vfmadd.s") ||
145 Name.startswith(
"fma.vfmadd.") ||
146 Name.startswith(
"fma.vfmsub.") ||
147 Name.startswith(
"fma.vfmsubadd.") ||
148 Name.startswith(
"fma.vfnmadd.") ||
149 Name.startswith(
"fma.vfnmsub.") ||
150 Name.startswith(
"avx512.mask.vfmadd.") ||
151 Name.startswith(
"avx512.mask.vfnmadd.") ||
152 Name.startswith(
"avx512.mask.vfnmsub.") ||
153 Name.startswith(
"avx512.mask3.vfmadd.") ||
154 Name.startswith(
"avx512.maskz.vfmadd.") ||
155 Name.startswith(
"avx512.mask3.vfmsub.") ||
156 Name.startswith(
"avx512.mask3.vfnmsub.") ||
157 Name.startswith(
"avx512.mask.vfmaddsub.") ||
158 Name.startswith(
"avx512.maskz.vfmaddsub.") ||
159 Name.startswith(
"avx512.mask3.vfmaddsub.") ||
160 Name.startswith(
"avx512.mask3.vfmsubadd.") ||
161 Name.startswith(
"avx512.mask.shuf.i") ||
162 Name.startswith(
"avx512.mask.shuf.f") ||
163 Name.startswith(
"avx512.kunpck") ||
164 Name.startswith(
"avx2.pabs.") ||
165 Name.startswith(
"avx512.mask.pabs.") ||
166 Name.startswith(
"avx512.broadcastm") ||
167 Name ==
"sse.sqrt.ss" ||
168 Name ==
"sse2.sqrt.sd" ||
169 Name.startswith(
"avx512.mask.sqrt.p") ||
170 Name.startswith(
"avx.sqrt.p") ||
171 Name.startswith(
"sse2.sqrt.p") ||
172 Name.startswith(
"sse.sqrt.p") ||
173 Name.startswith(
"avx512.mask.pbroadcast") ||
174 Name.startswith(
"sse2.pcmpeq.") ||
175 Name.startswith(
"sse2.pcmpgt.") ||
176 Name.startswith(
"avx2.pcmpeq.") ||
177 Name.startswith(
"avx2.pcmpgt.") ||
178 Name.startswith(
"avx512.mask.pcmpeq.") ||
179 Name.startswith(
"avx512.mask.pcmpgt.") ||
180 Name.startswith(
"avx.vperm2f128.") ||
181 Name ==
"avx2.vperm2i128" ||
182 Name ==
"sse.add.ss" ||
183 Name ==
"sse2.add.sd" ||
184 Name ==
"sse.sub.ss" ||
185 Name ==
"sse2.sub.sd" ||
186 Name ==
"sse.mul.ss" ||
187 Name ==
"sse2.mul.sd" ||
188 Name ==
"sse.div.ss" ||
189 Name ==
"sse2.div.sd" ||
190 Name ==
"sse41.pmaxsb" ||
191 Name ==
"sse2.pmaxs.w" ||
192 Name ==
"sse41.pmaxsd" ||
193 Name ==
"sse2.pmaxu.b" ||
194 Name ==
"sse41.pmaxuw" ||
195 Name ==
"sse41.pmaxud" ||
196 Name ==
"sse41.pminsb" ||
197 Name ==
"sse2.pmins.w" ||
198 Name ==
"sse41.pminsd" ||
199 Name ==
"sse2.pminu.b" ||
200 Name ==
"sse41.pminuw" ||
201 Name ==
"sse41.pminud" ||
202 Name ==
"avx512.kand.w" ||
203 Name ==
"avx512.kandn.w" ||
204 Name ==
"avx512.knot.w" ||
205 Name ==
"avx512.kor.w" ||
206 Name ==
"avx512.kxor.w" ||
207 Name ==
"avx512.kxnor.w" ||
208 Name ==
"avx512.kortestc.w" ||
209 Name ==
"avx512.kortestz.w" ||
210 Name.startswith(
"avx512.mask.pshuf.b.") ||
211 Name.startswith(
"avx2.pmax") ||
212 Name.startswith(
"avx2.pmin") ||
213 Name.startswith(
"avx512.mask.pmax") ||
214 Name.startswith(
"avx512.mask.pmin") ||
215 Name.startswith(
"avx2.vbroadcast") ||
216 Name.startswith(
"avx2.pbroadcast") ||
217 Name.startswith(
"avx.vpermil.") ||
218 Name.startswith(
"sse2.pshuf") ||
219 Name.startswith(
"avx512.pbroadcast") ||
220 Name.startswith(
"avx512.mask.broadcast.s") ||
221 Name.startswith(
"avx512.mask.movddup") ||
222 Name.startswith(
"avx512.mask.movshdup") ||
223 Name.startswith(
"avx512.mask.movsldup") ||
224 Name.startswith(
"avx512.mask.pshuf.d.") ||
225 Name.startswith(
"avx512.mask.pshufl.w.") ||
226 Name.startswith(
"avx512.mask.pshufh.w.") ||
227 Name.startswith(
"avx512.mask.shuf.p") ||
228 Name.startswith(
"avx512.mask.vpermil.p") ||
229 Name.startswith(
"avx512.mask.perm.df.") ||
230 Name.startswith(
"avx512.mask.perm.di.") ||
231 Name.startswith(
"avx512.mask.punpckl") ||
232 Name.startswith(
"avx512.mask.punpckh") ||
233 Name.startswith(
"avx512.mask.unpckl.") ||
234 Name.startswith(
"avx512.mask.unpckh.") ||
235 Name.startswith(
"avx512.mask.pand.") ||
236 Name.startswith(
"avx512.mask.pandn.") ||
237 Name.startswith(
"avx512.mask.por.") ||
238 Name.startswith(
"avx512.mask.pxor.") ||
239 Name.startswith(
"avx512.mask.and.") ||
240 Name.startswith(
"avx512.mask.andn.") ||
241 Name.startswith(
"avx512.mask.or.") ||
242 Name.startswith(
"avx512.mask.xor.") ||
243 Name.startswith(
"avx512.mask.padd.") ||
244 Name.startswith(
"avx512.mask.psub.") ||
245 Name.startswith(
"avx512.mask.pmull.") ||
246 Name.startswith(
"avx512.mask.cvtdq2pd.") ||
247 Name.startswith(
"avx512.mask.cvtudq2pd.") ||
248 Name.startswith(
"avx512.mask.cvtudq2ps.") ||
249 Name.startswith(
"avx512.mask.cvtqq2pd.") ||
250 Name.startswith(
"avx512.mask.cvtuqq2pd.") ||
251 Name.startswith(
"avx512.mask.cvtdq2ps.") ||
252 Name ==
"avx512.mask.vcvtph2ps.128" ||
253 Name ==
"avx512.mask.vcvtph2ps.256" ||
254 Name ==
"avx512.mask.cvtqq2ps.256" ||
255 Name ==
"avx512.mask.cvtqq2ps.512" ||
256 Name ==
"avx512.mask.cvtuqq2ps.256" ||
257 Name ==
"avx512.mask.cvtuqq2ps.512" ||
258 Name ==
"avx512.mask.cvtpd2dq.256" ||
259 Name ==
"avx512.mask.cvtpd2ps.256" ||
260 Name ==
"avx512.mask.cvttpd2dq.256" ||
261 Name ==
"avx512.mask.cvttps2dq.128" ||
262 Name ==
"avx512.mask.cvttps2dq.256" ||
263 Name ==
"avx512.mask.cvtps2pd.128" ||
264 Name ==
"avx512.mask.cvtps2pd.256" ||
265 Name ==
"avx512.cvtusi2sd" ||
266 Name.startswith(
"avx512.mask.permvar.") ||
267 Name ==
"sse2.pmulu.dq" ||
268 Name ==
"sse41.pmuldq" ||
269 Name ==
"avx2.pmulu.dq" ||
270 Name ==
"avx2.pmul.dq" ||
271 Name ==
"avx512.pmulu.dq.512" ||
272 Name ==
"avx512.pmul.dq.512" ||
273 Name.startswith(
"avx512.mask.pmul.dq.") ||
274 Name.startswith(
"avx512.mask.pmulu.dq.") ||
275 Name.startswith(
"avx512.mask.pmul.hr.sw.") ||
276 Name.startswith(
"avx512.mask.pmulh.w.") ||
277 Name.startswith(
"avx512.mask.pmulhu.w.") ||
278 Name.startswith(
"avx512.mask.pmaddw.d.") ||
279 Name.startswith(
"avx512.mask.pmaddubs.w.") ||
280 Name.startswith(
"avx512.mask.packsswb.") ||
281 Name.startswith(
"avx512.mask.packssdw.") ||
282 Name.startswith(
"avx512.mask.packuswb.") ||
283 Name.startswith(
"avx512.mask.packusdw.") ||
284 Name.startswith(
"avx512.mask.cmp.b") ||
285 Name.startswith(
"avx512.mask.cmp.d") ||
286 Name.startswith(
"avx512.mask.cmp.q") ||
287 Name.startswith(
"avx512.mask.cmp.w") ||
288 Name.startswith(
"avx512.cmp.p") ||
289 Name.startswith(
"avx512.mask.ucmp.") ||
290 Name.startswith(
"avx512.cvtb2mask.") ||
291 Name.startswith(
"avx512.cvtw2mask.") ||
292 Name.startswith(
"avx512.cvtd2mask.") ||
293 Name.startswith(
"avx512.cvtq2mask.") ||
294 Name.startswith(
"avx512.mask.vpermilvar.") ||
295 Name.startswith(
"avx512.mask.psll.d") ||
296 Name.startswith(
"avx512.mask.psll.q") ||
297 Name.startswith(
"avx512.mask.psll.w") ||
298 Name.startswith(
"avx512.mask.psra.d") ||
299 Name.startswith(
"avx512.mask.psra.q") ||
300 Name.startswith(
"avx512.mask.psra.w") ||
301 Name.startswith(
"avx512.mask.psrl.d") ||
302 Name.startswith(
"avx512.mask.psrl.q") ||
303 Name.startswith(
"avx512.mask.psrl.w") ||
304 Name.startswith(
"avx512.mask.pslli") ||
305 Name.startswith(
"avx512.mask.psrai") ||
306 Name.startswith(
"avx512.mask.psrli") ||
307 Name.startswith(
"avx512.mask.psllv") ||
308 Name.startswith(
"avx512.mask.psrav") ||
309 Name.startswith(
"avx512.mask.psrlv") ||
310 Name.startswith(
"sse41.pmovsx") ||
311 Name.startswith(
"sse41.pmovzx") ||
312 Name.startswith(
"avx2.pmovsx") ||
313 Name.startswith(
"avx2.pmovzx") ||
314 Name.startswith(
"avx512.mask.pmovsx") ||
315 Name.startswith(
"avx512.mask.pmovzx") ||
316 Name.startswith(
"avx512.mask.lzcnt.") ||
317 Name.startswith(
"avx512.mask.pternlog.") ||
318 Name.startswith(
"avx512.maskz.pternlog.") ||
319 Name.startswith(
"avx512.mask.vpmadd52") ||
320 Name.startswith(
"avx512.maskz.vpmadd52") ||
321 Name.startswith(
"avx512.mask.vpermi2var.") ||
322 Name.startswith(
"avx512.mask.vpermt2var.") ||
323 Name.startswith(
"avx512.maskz.vpermt2var.") ||
324 Name.startswith(
"avx512.mask.vpdpbusd.") ||
325 Name.startswith(
"avx512.maskz.vpdpbusd.") ||
326 Name.startswith(
"avx512.mask.vpdpbusds.") ||
327 Name.startswith(
"avx512.maskz.vpdpbusds.") ||
328 Name.startswith(
"avx512.mask.vpdpwssd.") ||
329 Name.startswith(
"avx512.maskz.vpdpwssd.") ||
330 Name.startswith(
"avx512.mask.vpdpwssds.") ||
331 Name.startswith(
"avx512.maskz.vpdpwssds.") ||
332 Name.startswith(
"avx512.mask.dbpsadbw.") ||
333 Name.startswith(
"avx512.mask.vpshld.") ||
334 Name.startswith(
"avx512.mask.vpshrd.") ||
335 Name.startswith(
"avx512.mask.vpshldv.") ||
336 Name.startswith(
"avx512.mask.vpshrdv.") ||
337 Name.startswith(
"avx512.maskz.vpshldv.") ||
338 Name.startswith(
"avx512.maskz.vpshrdv.") ||
339 Name.startswith(
"avx512.vpshld.") ||
340 Name.startswith(
"avx512.vpshrd.") ||
341 Name.startswith(
"avx512.mask.add.p") ||
342 Name.startswith(
"avx512.mask.sub.p") ||
343 Name.startswith(
"avx512.mask.mul.p") ||
344 Name.startswith(
"avx512.mask.div.p") ||
345 Name.startswith(
"avx512.mask.max.p") ||
346 Name.startswith(
"avx512.mask.min.p") ||
347 Name.startswith(
"avx512.mask.fpclass.p") ||
348 Name.startswith(
"avx512.mask.vpshufbitqmb.") ||
349 Name.startswith(
"avx512.mask.pmultishift.qb.") ||
350 Name.startswith(
"avx512.mask.conflict.") ||
351 Name ==
"avx512.mask.pmov.qd.256" ||
352 Name ==
"avx512.mask.pmov.qd.512" ||
353 Name ==
"avx512.mask.pmov.wb.256" ||
354 Name ==
"avx512.mask.pmov.wb.512" ||
355 Name ==
"sse.cvtsi2ss" ||
356 Name ==
"sse.cvtsi642ss" ||
357 Name ==
"sse2.cvtsi2sd" ||
358 Name ==
"sse2.cvtsi642sd" ||
359 Name ==
"sse2.cvtss2sd" ||
360 Name ==
"sse2.cvtdq2pd" ||
361 Name ==
"sse2.cvtdq2ps" ||
362 Name ==
"sse2.cvtps2pd" ||
363 Name ==
"avx.cvtdq2.pd.256" ||
364 Name ==
"avx.cvtdq2.ps.256" ||
365 Name ==
"avx.cvt.ps2.pd.256" ||
366 Name.startswith(
"vcvtph2ps.") ||
367 Name.startswith(
"avx.vinsertf128.") ||
368 Name ==
"avx2.vinserti128" ||
369 Name.startswith(
"avx512.mask.insert") ||
370 Name.startswith(
"avx.vextractf128.") ||
371 Name ==
"avx2.vextracti128" ||
372 Name.startswith(
"avx512.mask.vextract") ||
373 Name.startswith(
"sse4a.movnt.") ||
374 Name.startswith(
"avx.movnt.") ||
375 Name.startswith(
"avx512.storent.") ||
376 Name ==
"sse41.movntdqa" ||
377 Name ==
"avx2.movntdqa" ||
378 Name ==
"avx512.movntdqa" ||
379 Name ==
"sse2.storel.dq" ||
380 Name.startswith(
"sse.storeu.") ||
381 Name.startswith(
"sse2.storeu.") ||
382 Name.startswith(
"avx.storeu.") ||
383 Name.startswith(
"avx512.mask.storeu.") ||
384 Name.startswith(
"avx512.mask.store.p") ||
385 Name.startswith(
"avx512.mask.store.b.") ||
386 Name.startswith(
"avx512.mask.store.w.") ||
387 Name.startswith(
"avx512.mask.store.d.") ||
388 Name.startswith(
"avx512.mask.store.q.") ||
389 Name ==
"avx512.mask.store.ss" ||
390 Name.startswith(
"avx512.mask.loadu.") ||
391 Name.startswith(
"avx512.mask.load.") ||
392 Name.startswith(
"avx512.mask.expand.load.") ||
393 Name.startswith(
"avx512.mask.compress.store.") ||
394 Name.startswith(
"avx512.mask.expand.b") ||
395 Name.startswith(
"avx512.mask.expand.w") ||
396 Name.startswith(
"avx512.mask.expand.d") ||
397 Name.startswith(
"avx512.mask.expand.q") ||
398 Name.startswith(
"avx512.mask.expand.p") ||
399 Name.startswith(
"avx512.mask.compress.b") ||
400 Name.startswith(
"avx512.mask.compress.w") ||
401 Name.startswith(
"avx512.mask.compress.d") ||
402 Name.startswith(
"avx512.mask.compress.q") ||
403 Name.startswith(
"avx512.mask.compress.p") ||
404 Name ==
"sse42.crc32.64.8" ||
405 Name.startswith(
"avx.vbroadcast.s") ||
406 Name.startswith(
"avx512.vbroadcast.s") ||
407 Name.startswith(
"avx512.mask.palignr.") ||
408 Name.startswith(
"avx512.mask.valign.") ||
409 Name.startswith(
"sse2.psll.dq") ||
410 Name.startswith(
"sse2.psrl.dq") ||
411 Name.startswith(
"avx2.psll.dq") ||
412 Name.startswith(
"avx2.psrl.dq") ||
413 Name.startswith(
"avx512.psll.dq") ||
414 Name.startswith(
"avx512.psrl.dq") ||
415 Name ==
"sse41.pblendw" ||
416 Name.startswith(
"sse41.blendp") ||
417 Name.startswith(
"avx.blend.p") ||
418 Name ==
"avx2.pblendw" ||
419 Name.startswith(
"avx2.pblendd.") ||
420 Name.startswith(
"avx.vbroadcastf128") ||
421 Name ==
"avx2.vbroadcasti128" ||
422 Name.startswith(
"avx512.mask.broadcastf32x4.") ||
423 Name.startswith(
"avx512.mask.broadcastf64x2.") ||
424 Name.startswith(
"avx512.mask.broadcastf32x8.") ||
425 Name.startswith(
"avx512.mask.broadcastf64x4.") ||
426 Name.startswith(
"avx512.mask.broadcasti32x4.") ||
427 Name.startswith(
"avx512.mask.broadcasti64x2.") ||
428 Name.startswith(
"avx512.mask.broadcasti32x8.") ||
429 Name.startswith(
"avx512.mask.broadcasti64x4.") ||
430 Name ==
"xop.vpcmov" ||
431 Name ==
"xop.vpcmov.256" ||
432 Name.startswith(
"avx512.mask.move.s") ||
433 Name.startswith(
"avx512.cvtmask2") ||
434 Name.startswith(
"xop.vpcom") ||
435 Name.startswith(
"xop.vprot") ||
436 Name.startswith(
"avx512.prol") ||
437 Name.startswith(
"avx512.pror") ||
438 Name.startswith(
"avx512.mask.prorv.") ||
439 Name.startswith(
"avx512.mask.pror.") ||
440 Name.startswith(
"avx512.mask.prolv.") ||
441 Name.startswith(
"avx512.mask.prol.") ||
442 Name.startswith(
"avx512.ptestm") ||
443 Name.startswith(
"avx512.ptestnm") ||
444 Name.startswith(
"avx512.mask.pavg"))
453 if (!
Name.startswith(
"x86."))
463 if (
Name ==
"rdtscp") {
465 if (
F->getFunctionType()->getNumParams() == 0)
470 Intrinsic::x86_rdtscp);
475 if (
Name.startswith(
"sse41.ptest")) {
476 if (
Name.substr(11) ==
"c")
478 if (
Name.substr(11) ==
"z")
480 if (
Name.substr(11) ==
"nzc")
485 if (
Name ==
"sse41.insertps")
488 if (
Name ==
"sse41.dppd")
491 if (
Name ==
"sse41.dpps")
494 if (
Name ==
"sse41.mpsadbw")
497 if (
Name ==
"avx.dp.ps.256")
500 if (
Name ==
"avx2.mpsadbw")
503 if (
Name ==
"avx512.mask.cmp.pd.128")
506 if (
Name ==
"avx512.mask.cmp.pd.256")
509 if (
Name ==
"avx512.mask.cmp.pd.512")
512 if (
Name ==
"avx512.mask.cmp.ps.128")
515 if (
Name ==
"avx512.mask.cmp.ps.256")
518 if (
Name ==
"avx512.mask.cmp.ps.512")
521 if (
Name ==
"avx512bf16.cvtne2ps2bf16.128")
523 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128, NewFn);
524 if (
Name ==
"avx512bf16.cvtne2ps2bf16.256")
526 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256, NewFn);
527 if (
Name ==
"avx512bf16.cvtne2ps2bf16.512")
529 F, Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512, NewFn);
530 if (
Name ==
"avx512bf16.mask.cvtneps2bf16.128")
532 F, Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, NewFn);
533 if (
Name ==
"avx512bf16.cvtneps2bf16.256")
535 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_256, NewFn);
536 if (
Name ==
"avx512bf16.cvtneps2bf16.512")
538 F, Intrinsic::x86_avx512bf16_cvtneps2bf16_512, NewFn);
539 if (
Name ==
"avx512bf16.dpbf16ps.128")
541 F, Intrinsic::x86_avx512bf16_dpbf16ps_128, NewFn);
542 if (
Name ==
"avx512bf16.dpbf16ps.256")
544 F, Intrinsic::x86_avx512bf16_dpbf16ps_256, NewFn);
545 if (
Name ==
"avx512bf16.dpbf16ps.512")
547 F, Intrinsic::x86_avx512bf16_dpbf16ps_512, NewFn);
550 if (
Name.startswith(
"xop.vfrcz.ss") &&
F->arg_size() == 2) {
553 Intrinsic::x86_xop_vfrcz_ss);
556 if (
Name.startswith(
"xop.vfrcz.sd") &&
F->arg_size() == 2) {
559 Intrinsic::x86_xop_vfrcz_sd);
563 if (
Name.startswith(
"xop.vpermil2")) {
564 auto Idx =
F->getFunctionType()->getParamType(2);
565 if (
Idx->isFPOrFPVectorTy()) {
567 unsigned IdxSize =
Idx->getPrimitiveSizeInBits();
568 unsigned EltSize =
Idx->getScalarSizeInBits();
570 if (EltSize == 64 && IdxSize == 128)
571 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
572 else if (EltSize == 32 && IdxSize == 128)
573 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
574 else if (EltSize == 64 && IdxSize == 256)
575 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
577 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
583 if (
Name ==
"seh.recoverfp") {
592 assert(
F &&
"Illegal to upgrade a non-existent Function.");
596 if (
Name.size() <= 7 || !
Name.startswith(
"llvm."))
603 if (
Name.startswith(
"arm.rbit") ||
Name.startswith(
"aarch64.rbit")) {
605 F->arg_begin()->getType());
608 if (
Name.startswith(
"aarch64.neon.frintn")) {
610 F->arg_begin()->getType());
613 if (
Name.startswith(
"aarch64.neon.rbit")) {
615 F->arg_begin()->getType());
618 if (
Name ==
"aarch64.sve.bfdot.lane") {
620 Intrinsic::aarch64_sve_bfdot_lane_v2);
623 if (
Name ==
"aarch64.sve.bfmlalb.lane") {
625 Intrinsic::aarch64_sve_bfmlalb_lane_v2);
628 if (
Name ==
"aarch64.sve.bfmlalt.lane") {
630 Intrinsic::aarch64_sve_bfmlalt_lane_v2);
633 static const Regex LdRegex(
"^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
636 dyn_cast<VectorType>(
F->getReturnType())->getElementType();
638 dyn_cast<VectorType>(
F->arg_begin()->getType())->getElementCount();
639 Type *Ty = VectorType::get(ScalarTy, EC);
642 .
StartsWith(
"aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
643 .
StartsWith(
"aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
644 .
StartsWith(
"aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
649 if (
Name.startswith(
"aarch64.sve.tuple.get")) {
650 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
652 Intrinsic::vector_extract, Tys);
655 if (
Name.startswith(
"aarch64.sve.tuple.set")) {
656 auto Args =
F->getFunctionType()->params();
657 Type *Tys[] = {Args[0], Args[2], Args[1]};
659 Intrinsic::vector_insert, Tys);
662 static const Regex CreateTupleRegex(
663 "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
665 auto Args =
F->getFunctionType()->params();
666 Type *Tys[] = {
F->getReturnType(), Args[1]};
668 Intrinsic::vector_insert, Tys);
671 if (
Name.startswith(
"arm.neon.vclz")) {
673 F->arg_begin()->getType(),
681 "llvm.ctlz." +
Name.substr(14),
F->getParent());
684 if (
Name.startswith(
"arm.neon.vcnt")) {
686 F->arg_begin()->getType());
689 static const Regex vstRegex(
"^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
691 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
692 Intrinsic::arm_neon_vst2,
693 Intrinsic::arm_neon_vst3,
694 Intrinsic::arm_neon_vst4};
697 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
698 Intrinsic::arm_neon_vst4lane
701 auto fArgs =
F->getFunctionType()->params();
702 Type *Tys[] = {fArgs[0], fArgs[1]};
703 if (!
Name.contains(
"lane"))
705 StoreInts[fArgs.size() - 3], Tys);
708 StoreLaneInts[fArgs.size() - 5], Tys);
711 if (
Name ==
"aarch64.thread.pointer" ||
Name ==
"arm.thread.pointer") {
715 if (
Name.startswith(
"arm.neon.vqadds.")) {
717 F->arg_begin()->getType());
720 if (
Name.startswith(
"arm.neon.vqaddu.")) {
722 F->arg_begin()->getType());
725 if (
Name.startswith(
"arm.neon.vqsubs.")) {
727 F->arg_begin()->getType());
730 if (
Name.startswith(
"arm.neon.vqsubu.")) {
732 F->arg_begin()->getType());
735 if (
Name.startswith(
"aarch64.neon.addp")) {
736 if (
F->arg_size() != 2)
738 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
739 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
741 Intrinsic::aarch64_neon_faddp, Ty);
748 if ((
Name.startswith(
"arm.neon.bfdot.") ||
749 Name.startswith(
"aarch64.neon.bfdot.")) &&
750 Name.endswith(
"i8")) {
753 .
Cases(
"arm.neon.bfdot.v2f32.v8i8",
754 "arm.neon.bfdot.v4f32.v16i8",
755 Intrinsic::arm_neon_bfdot)
756 .
Cases(
"aarch64.neon.bfdot.v2f32.v8i8",
757 "aarch64.neon.bfdot.v4f32.v16i8",
758 Intrinsic::aarch64_neon_bfdot)
763 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
764 assert((OperandWidth == 64 || OperandWidth == 128) &&
765 "Unexpected operand width");
767 std::array<Type *, 2> Tys {{
777 if ((
Name.startswith(
"arm.neon.bfm") ||
778 Name.startswith(
"aarch64.neon.bfm")) &&
779 Name.endswith(
".v4f32.v16i8")) {
782 .
Case(
"arm.neon.bfmmla.v4f32.v16i8",
783 Intrinsic::arm_neon_bfmmla)
784 .
Case(
"arm.neon.bfmlalb.v4f32.v16i8",
785 Intrinsic::arm_neon_bfmlalb)
786 .
Case(
"arm.neon.bfmlalt.v4f32.v16i8",
787 Intrinsic::arm_neon_bfmlalt)
788 .
Case(
"aarch64.neon.bfmmla.v4f32.v16i8",
789 Intrinsic::aarch64_neon_bfmmla)
790 .
Case(
"aarch64.neon.bfmlalb.v4f32.v16i8",
791 Intrinsic::aarch64_neon_bfmlalb)
792 .
Case(
"aarch64.neon.bfmlalt.v4f32.v16i8",
793 Intrinsic::aarch64_neon_bfmlalt)
798 std::array<Type *, 0> Tys;
803 if (
Name ==
"arm.mve.vctp64" &&
804 cast<FixedVectorType>(
F->getReturnType())->getNumElements() == 4) {
811 if (
Name ==
"arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
812 Name ==
"arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
813 Name ==
"arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
814 Name ==
"arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
816 "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
817 Name ==
"arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
818 Name ==
"arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
819 Name ==
"arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
821 "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
822 Name ==
"arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
823 Name ==
"arm.cde.vcx1q.predicated.v2i64.v4i1" ||
824 Name ==
"arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
825 Name ==
"arm.cde.vcx2q.predicated.v2i64.v4i1" ||
826 Name ==
"arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
827 Name ==
"arm.cde.vcx3q.predicated.v2i64.v4i1" ||
828 Name ==
"arm.cde.vcx3qa.predicated.v2i64.v4i1")
831 if (
Name ==
"amdgcn.alignbit") {
834 {F->getReturnType()});
842 if (
Name.startswith(
"ctlz.") &&
F->arg_size() == 1) {
845 F->arg_begin()->getType());
848 if (
Name.startswith(
"cttz.") &&
F->arg_size() == 1) {
851 F->arg_begin()->getType());
857 if (
Name ==
"dbg.addr") {
862 if (
Name ==
"dbg.value" &&
F->arg_size() == 4) {
870 if (
Name.startswith(
"experimental.vector.extract.")) {
872 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
874 Intrinsic::vector_extract, Tys);
878 if (
Name.startswith(
"experimental.vector.insert.")) {
880 auto Args =
F->getFunctionType()->params();
881 Type *Tys[] = {Args[0], Args[1]};
883 Intrinsic::vector_insert, Tys);
888 static const Regex R(
"^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
892 .
Case(
"add", Intrinsic::vector_reduce_add)
893 .
Case(
"mul", Intrinsic::vector_reduce_mul)
894 .
Case(
"and", Intrinsic::vector_reduce_and)
895 .
Case(
"or", Intrinsic::vector_reduce_or)
896 .
Case(
"xor", Intrinsic::vector_reduce_xor)
897 .
Case(
"smax", Intrinsic::vector_reduce_smax)
898 .
Case(
"smin", Intrinsic::vector_reduce_smin)
899 .
Case(
"umax", Intrinsic::vector_reduce_umax)
900 .
Case(
"umin", Intrinsic::vector_reduce_umin)
901 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
902 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
906 auto Args =
F->getFunctionType()->params();
912 "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
917 ID = Intrinsic::vector_reduce_fadd;
919 ID = Intrinsic::vector_reduce_fmul;
922 auto Args =
F->getFunctionType()->params();
923 Type *Tys[] = {Args[1]};
931 if (
Name.startswith(
"flt.rounds")) {
939 bool IsLifetimeStart =
Name.startswith(
"lifetime.start");
940 if (IsLifetimeStart ||
Name.startswith(
"invariant.start")) {
942 Intrinsic::lifetime_start : Intrinsic::invariant_start;
943 auto Args =
F->getFunctionType()->params();
944 Type* ObjectPtr[1] = {Args[1]};
952 bool IsLifetimeEnd =
Name.startswith(
"lifetime.end");
953 if (IsLifetimeEnd ||
Name.startswith(
"invariant.end")) {
955 Intrinsic::lifetime_end : Intrinsic::invariant_end;
957 auto Args =
F->getFunctionType()->params();
958 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
965 if (
Name.startswith(
"invariant.group.barrier")) {
967 auto Args =
F->getFunctionType()->params();
968 Type* ObjectPtr[1] = {Args[0]};
971 Intrinsic::launder_invariant_group, ObjectPtr);
979 if (
Name.startswith(
"masked.load.")) {
980 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType() };
985 Intrinsic::masked_load,
990 if (
Name.startswith(
"masked.store.")) {
991 auto Args =
F->getFunctionType()->params();
992 Type *Tys[] = { Args[0], Args[1] };
997 Intrinsic::masked_store,
1004 if (
Name.startswith(
"masked.gather.")) {
1005 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
1010 Intrinsic::masked_gather, Tys);
1014 if (
Name.startswith(
"masked.scatter.")) {
1015 auto Args =
F->getFunctionType()->params();
1016 Type *Tys[] = {Args[0], Args[1]};
1021 Intrinsic::masked_scatter, Tys);
1028 if (
Name.startswith(
"memcpy.") &&
F->arg_size() == 5) {
1036 if (
Name.startswith(
"memmove.") &&
F->arg_size() == 5) {
1044 if (
Name.startswith(
"memset.") &&
F->arg_size() == 5) {
1047 const auto *FT =
F->getFunctionType();
1048 Type *ParamTypes[2] = {
1049 FT->getParamType(0),
1059 if (
Name.startswith(
"nvvm.")) {
1064 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
1065 .
Case(
"clz.i", Intrinsic::ctlz)
1066 .
Case(
"popc.i", Intrinsic::ctpop)
1070 {F->getReturnType()});
1079 .
Cases(
"abs.i",
"abs.ll",
true)
1080 .
Cases(
"clz.ll",
"popc.ll",
"h2f",
true)
1081 .
Cases(
"max.i",
"max.ll",
"max.ui",
"max.ull",
true)
1082 .
Cases(
"min.i",
"min.ll",
"min.ui",
"min.ull",
true)
1096 if (
Name.startswith(
"objectsize.")) {
1097 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
1098 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
1110 if (
Name ==
"prefetch") {
1112 Type *Tys[] = {
F->arg_begin()->getType()};
1120 }
else if (
Name.startswith(
"ptr.annotation.") &&
F->arg_size() == 4) {
1123 F->getParent(), Intrinsic::ptr_annotation,
1124 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1130 if (
Name ==
"stackprotectorcheck") {
1137 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
1140 F->getParent(), Intrinsic::var_annotation,
1141 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1148 if (
Name.startswith(
"wasm.fma.")) {
1151 F->getParent(), Intrinsic::wasm_relaxed_madd,
F->getReturnType());
1154 if (
Name.startswith(
"wasm.fms.")) {
1157 F->getParent(), Intrinsic::wasm_relaxed_nmadd,
F->getReturnType());
1160 if (
Name.startswith(
"wasm.laneselect.")) {
1163 F->getParent(), Intrinsic::wasm_relaxed_laneselect,
1164 F->getReturnType());
1167 if (
Name ==
"wasm.dot.i8x16.i7x16.signed") {
1170 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed);
1173 if (
Name ==
"wasm.dot.i8x16.i7x16.add.signed") {
1176 F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
1186 auto *
ST = dyn_cast<StructType>(
F->getReturnType());
1187 if (ST && (!
ST->isLiteral() ||
ST->isPacked())) {
1195 auto *FT =
F->getFunctionType();
1197 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1198 std::string
Name =
F->getName().str();
1201 Name,
F->getParent());
1212 if (Result != std::nullopt) {
1227 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1239 GV->
getName() ==
"llvm.global_dtors")) ||
1254 unsigned N =
Init->getNumOperands();
1255 std::vector<Constant *> NewCtors(
N);
1256 for (
unsigned i = 0; i !=
N; ++i) {
1257 auto Ctor = cast<Constant>(
Init->getOperand(i));
1259 EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1271 Value *Op,
unsigned Shift) {
1272 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1273 unsigned NumElts = ResultTy->getNumElements() * 8;
1277 Op =
Builder.CreateBitCast(Op, VecTy,
"cast");
1287 for (
unsigned l = 0; l != NumElts; l += 16)
1288 for (
unsigned i = 0; i != 16; ++i) {
1289 unsigned Idx = NumElts + i - Shift;
1291 Idx -= NumElts - 16;
1292 Idxs[l + i] =
Idx + l;
1299 return Builder.CreateBitCast(Res, ResultTy,
"cast");
1306 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1307 unsigned NumElts = ResultTy->getNumElements() * 8;
1311 Op =
Builder.CreateBitCast(Op, VecTy,
"cast");
1321 for (
unsigned l = 0; l != NumElts; l += 16)
1322 for (
unsigned i = 0; i != 16; ++i) {
1323 unsigned Idx = i + Shift;
1325 Idx += NumElts - 16;
1326 Idxs[l + i] =
Idx + l;
1333 return Builder.CreateBitCast(Res, ResultTy,
"cast");
1340 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1341 Mask =
Builder.CreateBitCast(Mask, MaskTy);
1347 for (
unsigned i = 0; i != NumElts; ++i)
1349 Mask =
Builder.CreateShuffleVector(Mask, Mask,
ArrayRef(Indices, NumElts),
1359 if (
const auto *
C = dyn_cast<Constant>(Mask))
1360 if (
C->isAllOnesValue())
1364 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1365 return Builder.CreateSelect(Mask, Op0, Op1);
1371 if (
const auto *
C = dyn_cast<Constant>(Mask))
1372 if (
C->isAllOnesValue())
1376 Mask->getType()->getIntegerBitWidth());
1377 Mask =
Builder.CreateBitCast(Mask, MaskTy);
1379 return Builder.CreateSelect(Mask, Op0, Op1);
1389 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1391 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1392 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1393 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1398 ShiftVal &= (NumElts - 1);
1407 if (ShiftVal > 16) {
1415 for (
unsigned l = 0; l < NumElts; l += 16) {
1416 for (
unsigned i = 0; i != 16; ++i) {
1417 unsigned Idx = ShiftVal + i;
1418 if (!IsVALIGN &&
Idx >= 16)
1419 Idx += NumElts - 16;
1420 Indices[l + i] =
Idx + l;
1425 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
1431 bool ZeroMask,
bool IndexForm) {
1437 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1438 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1439 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1440 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1441 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1442 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1443 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1444 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1445 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1446 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1447 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1448 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1449 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1450 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1451 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1452 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1453 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1454 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1455 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1456 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1457 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1458 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1459 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1460 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1461 else if (VecWidth == 128 && EltWidth == 16)
1462 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1463 else if (VecWidth == 256 && EltWidth == 16)
1464 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1465 else if (VecWidth == 512 && EltWidth == 16)
1466 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1467 else if (VecWidth == 128 && EltWidth == 8)
1468 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1469 else if (VecWidth == 256 && EltWidth == 8)
1470 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1471 else if (VecWidth == 512 && EltWidth == 8)
1472 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1508 bool IsRotateRight) {
1517 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1519 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
1522 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1524 Value *Res =
Builder.CreateCall(Intrin, {Src, Src, Amt});
1543 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1546 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1549 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1552 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1555 Pred = ICmpInst::ICMP_EQ;
1558 Pred = ICmpInst::ICMP_NE;
1574 bool IsShiftRight,
bool ZeroMask) {
1587 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1589 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
1592 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1594 Value *Res =
Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1613 const Align Alignment =
1615 ?
Align(
Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1619 if (
const auto *
C = dyn_cast<Constant>(Mask))
1620 if (
C->isAllOnesValue())
1624 unsigned NumElts = cast<FixedVectorType>(
Data->getType())->getNumElements();
1635 const Align Alignment =
1643 if (
const auto *
C = dyn_cast<Constant>(Mask))
1644 if (
C->isAllOnesValue())
1645 return Builder.CreateAlignedLoad(ValTy,
Ptr, Alignment);
1648 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1650 return Builder.CreateMaskedLoad(ValTy,
Ptr, Alignment, Mask, Passthru);
1695 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1697 const auto *
C = dyn_cast<Constant>(Mask);
1698 if (!
C || !
C->isAllOnesValue())
1704 for (
unsigned i = 0; i != NumElts; ++i)
1706 for (
unsigned i = NumElts; i != 8; ++i)
1707 Indices[i] = NumElts + i % NumElts;
1708 Vec =
Builder.CreateShuffleVector(Vec,
1712 return Builder.CreateBitCast(Vec,
Builder.getIntNTy(std::max(NumElts, 8U)));
1718 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1724 }
else if (
CC == 7) {
1731 case 0: Pred = ICmpInst::ICMP_EQ;
break;
1732 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
1733 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
1734 case 4: Pred = ICmpInst::ICMP_NE;
break;
1735 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
1736 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
1773 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
1775 return Builder.CreateSExt(Mask, ReturnOp,
"vpmovm2");
1786 if (
Name.startswith(
"max.p")) {
1787 if (VecWidth == 128 && EltWidth == 32)
1788 IID = Intrinsic::x86_sse_max_ps;
1789 else if (VecWidth == 128 && EltWidth == 64)
1790 IID = Intrinsic::x86_sse2_max_pd;
1791 else if (VecWidth == 256 && EltWidth == 32)
1792 IID = Intrinsic::x86_avx_max_ps_256;
1793 else if (VecWidth == 256 && EltWidth == 64)
1794 IID = Intrinsic::x86_avx_max_pd_256;
1797 }
else if (
Name.startswith(
"min.p")) {
1798 if (VecWidth == 128 && EltWidth == 32)
1799 IID = Intrinsic::x86_sse_min_ps;
1800 else if (VecWidth == 128 && EltWidth == 64)
1801 IID = Intrinsic::x86_sse2_min_pd;
1802 else if (VecWidth == 256 && EltWidth == 32)
1803 IID = Intrinsic::x86_avx_min_ps_256;
1804 else if (VecWidth == 256 && EltWidth == 64)
1805 IID = Intrinsic::x86_avx_min_pd_256;
1808 }
else if (
Name.startswith(
"pshuf.b.")) {
1809 if (VecWidth == 128)
1810 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1811 else if (VecWidth == 256)
1812 IID = Intrinsic::x86_avx2_pshuf_b;
1813 else if (VecWidth == 512)
1814 IID = Intrinsic::x86_avx512_pshuf_b_512;
1817 }
else if (
Name.startswith(
"pmul.hr.sw.")) {
1818 if (VecWidth == 128)
1819 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1820 else if (VecWidth == 256)
1821 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1822 else if (VecWidth == 512)
1823 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1826 }
else if (
Name.startswith(
"pmulh.w.")) {
1827 if (VecWidth == 128)
1828 IID = Intrinsic::x86_sse2_pmulh_w;
1829 else if (VecWidth == 256)
1830 IID = Intrinsic::x86_avx2_pmulh_w;
1831 else if (VecWidth == 512)
1832 IID = Intrinsic::x86_avx512_pmulh_w_512;
1835 }
else if (
Name.startswith(
"pmulhu.w.")) {
1836 if (VecWidth == 128)
1837 IID = Intrinsic::x86_sse2_pmulhu_w;
1838 else if (VecWidth == 256)
1839 IID = Intrinsic::x86_avx2_pmulhu_w;
1840 else if (VecWidth == 512)
1841 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1844 }
else if (
Name.startswith(
"pmaddw.d.")) {
1845 if (VecWidth == 128)
1846 IID = Intrinsic::x86_sse2_pmadd_wd;
1847 else if (VecWidth == 256)
1848 IID = Intrinsic::x86_avx2_pmadd_wd;
1849 else if (VecWidth == 512)
1850 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1853 }
else if (
Name.startswith(
"pmaddubs.w.")) {
1854 if (VecWidth == 128)
1855 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1856 else if (VecWidth == 256)
1857 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1858 else if (VecWidth == 512)
1859 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1862 }
else if (
Name.startswith(
"packsswb.")) {
1863 if (VecWidth == 128)
1864 IID = Intrinsic::x86_sse2_packsswb_128;
1865 else if (VecWidth == 256)
1866 IID = Intrinsic::x86_avx2_packsswb;
1867 else if (VecWidth == 512)
1868 IID = Intrinsic::x86_avx512_packsswb_512;
1871 }
else if (
Name.startswith(
"packssdw.")) {
1872 if (VecWidth == 128)
1873 IID = Intrinsic::x86_sse2_packssdw_128;
1874 else if (VecWidth == 256)
1875 IID = Intrinsic::x86_avx2_packssdw;
1876 else if (VecWidth == 512)
1877 IID = Intrinsic::x86_avx512_packssdw_512;
1880 }
else if (
Name.startswith(
"packuswb.")) {
1881 if (VecWidth == 128)
1882 IID = Intrinsic::x86_sse2_packuswb_128;
1883 else if (VecWidth == 256)
1884 IID = Intrinsic::x86_avx2_packuswb;
1885 else if (VecWidth == 512)
1886 IID = Intrinsic::x86_avx512_packuswb_512;
1889 }
else if (
Name.startswith(
"packusdw.")) {
1890 if (VecWidth == 128)
1891 IID = Intrinsic::x86_sse41_packusdw;
1892 else if (VecWidth == 256)
1893 IID = Intrinsic::x86_avx2_packusdw;
1894 else if (VecWidth == 512)
1895 IID = Intrinsic::x86_avx512_packusdw_512;
1898 }
else if (
Name.startswith(
"vpermilvar.")) {
1899 if (VecWidth == 128 && EltWidth == 32)
1900 IID = Intrinsic::x86_avx_vpermilvar_ps;
1901 else if (VecWidth == 128 && EltWidth == 64)
1902 IID = Intrinsic::x86_avx_vpermilvar_pd;
1903 else if (VecWidth == 256 && EltWidth == 32)
1904 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1905 else if (VecWidth == 256 && EltWidth == 64)
1906 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1907 else if (VecWidth == 512 && EltWidth == 32)
1908 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1909 else if (VecWidth == 512 && EltWidth == 64)
1910 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1913 }
else if (
Name ==
"cvtpd2dq.256") {
1914 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1915 }
else if (
Name ==
"cvtpd2ps.256") {
1916 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1917 }
else if (
Name ==
"cvttpd2dq.256") {
1918 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1919 }
else if (
Name ==
"cvttps2dq.128") {
1920 IID = Intrinsic::x86_sse2_cvttps2dq;
1921 }
else if (
Name ==
"cvttps2dq.256") {
1922 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1923 }
else if (
Name.startswith(
"permvar.")) {
1925 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1926 IID = Intrinsic::x86_avx2_permps;
1927 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1928 IID = Intrinsic::x86_avx2_permd;
1929 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1930 IID = Intrinsic::x86_avx512_permvar_df_256;
1931 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1932 IID = Intrinsic::x86_avx512_permvar_di_256;
1933 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1934 IID = Intrinsic::x86_avx512_permvar_sf_512;
1935 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1936 IID = Intrinsic::x86_avx512_permvar_si_512;
1937 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1938 IID = Intrinsic::x86_avx512_permvar_df_512;
1939 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1940 IID = Intrinsic::x86_avx512_permvar_di_512;
1941 else if (VecWidth == 128 && EltWidth == 16)
1942 IID = Intrinsic::x86_avx512_permvar_hi_128;
1943 else if (VecWidth == 256 && EltWidth == 16)
1944 IID = Intrinsic::x86_avx512_permvar_hi_256;
1945 else if (VecWidth == 512 && EltWidth == 16)
1946 IID = Intrinsic::x86_avx512_permvar_hi_512;
1947 else if (VecWidth == 128 && EltWidth == 8)
1948 IID = Intrinsic::x86_avx512_permvar_qi_128;
1949 else if (VecWidth == 256 && EltWidth == 8)
1950 IID = Intrinsic::x86_avx512_permvar_qi_256;
1951 else if (VecWidth == 512 && EltWidth == 8)
1952 IID = Intrinsic::x86_avx512_permvar_qi_512;
1955 }
else if (
Name.startswith(
"dbpsadbw.")) {
1956 if (VecWidth == 128)
1957 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1958 else if (VecWidth == 256)
1959 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1960 else if (VecWidth == 512)
1961 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1964 }
else if (
Name.startswith(
"pmultishift.qb.")) {
1965 if (VecWidth == 128)
1966 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1967 else if (VecWidth == 256)
1968 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1969 else if (VecWidth == 512)
1970 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1973 }
else if (
Name.startswith(
"conflict.")) {
1974 if (
Name[9] ==
'd' && VecWidth == 128)
1975 IID = Intrinsic::x86_avx512_conflict_d_128;
1976 else if (
Name[9] ==
'd' && VecWidth == 256)
1977 IID = Intrinsic::x86_avx512_conflict_d_256;
1978 else if (
Name[9] ==
'd' && VecWidth == 512)
1979 IID = Intrinsic::x86_avx512_conflict_d_512;
1980 else if (
Name[9] ==
'q' && VecWidth == 128)
1981 IID = Intrinsic::x86_avx512_conflict_q_128;
1982 else if (
Name[9] ==
'q' && VecWidth == 256)
1983 IID = Intrinsic::x86_avx512_conflict_q_256;
1984 else if (
Name[9] ==
'q' && VecWidth == 512)
1985 IID = Intrinsic::x86_avx512_conflict_q_512;
1988 }
else if (
Name.startswith(
"pavg.")) {
1989 if (
Name[5] ==
'b' && VecWidth == 128)
1990 IID = Intrinsic::x86_sse2_pavg_b;
1991 else if (
Name[5] ==
'b' && VecWidth == 256)
1992 IID = Intrinsic::x86_avx2_pavg_b;
1993 else if (
Name[5] ==
'b' && VecWidth == 512)
1994 IID = Intrinsic::x86_avx512_pavg_b_512;
1995 else if (
Name[5] ==
'w' && VecWidth == 128)
1996 IID = Intrinsic::x86_sse2_pavg_w;
1997 else if (
Name[5] ==
'w' && VecWidth == 256)
1998 IID = Intrinsic::x86_avx2_pavg_w;
1999 else if (
Name[5] ==
'w' && VecWidth == 512)
2000 IID = Intrinsic::x86_avx512_pavg_w_512;
2021 if (AsmStr->find(
"mov\tfp") == 0 &&
2022 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
2023 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
2024 AsmStr->replace(Pos, 1,
";");
2030 if (
Name ==
"mve.vctp64.old") {
2038 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2039 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2043 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2044 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2046 }
else if (
Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2047 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2048 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2049 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2051 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2052 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2053 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2054 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2056 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2057 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2058 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
2059 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
2060 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
2061 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
2062 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
2063 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
2064 std::vector<Type *> Tys;
2068 case Intrinsic::arm_mve_mull_int_predicated:
2069 case Intrinsic::arm_mve_vqdmull_predicated:
2070 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2073 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2074 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2075 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2079 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2083 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2087 case Intrinsic::arm_cde_vcx1q_predicated:
2088 case Intrinsic::arm_cde_vcx1qa_predicated:
2089 case Intrinsic::arm_cde_vcx2q_predicated:
2090 case Intrinsic::arm_cde_vcx2qa_predicated:
2091 case Intrinsic::arm_cde_vcx3q_predicated:
2092 case Intrinsic::arm_cde_vcx3qa_predicated:
2099 std::vector<Value *> Ops;
2101 Type *Ty = Op->getType();
2105 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2106 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2110 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2140 assert(
Name.startswith(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
2143 bool IsX86 =
Name.startswith(
"x86.");
2146 bool IsNVVM =
Name.startswith(
"nvvm.");
2149 bool IsARM =
Name.startswith(
"arm.");
2153 if (IsX86 &&
Name.startswith(
"sse4a.movnt.")) {
2165 Type *SrcEltTy = cast<VectorType>(Arg1->
getType())->getElementType();
2166 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2172 SI->setMetadata(M->getMDKindID(
"nontemporal"), Node);
2179 if (IsX86 && (
Name.startswith(
"avx.movnt.") ||
2180 Name.startswith(
"avx512.storent."))) {
2192 PointerType::getUnqual(Arg1->
getType()),
2197 SI->setMetadata(M->getMDKindID(
"nontemporal"), Node);
2204 if (IsX86 &&
Name ==
"sse2.storel.dq") {
2209 Value *BC0 =
Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
2212 PointerType::getUnqual(Elt->
getType()),
2221 if (IsX86 && (
Name.startswith(
"sse.storeu.") ||
2222 Name.startswith(
"sse2.storeu.") ||
2223 Name.startswith(
"avx.storeu."))) {
2227 Arg0 =
Builder.CreateBitCast(Arg0,
2228 PointerType::getUnqual(Arg1->
getType()),
2237 if (IsX86 &&
Name ==
"avx512.mask.store.ss") {
2247 if (IsX86 && (
Name.startswith(
"avx512.mask.store"))) {
2260 if (IsX86 && (
Name.startswith(
"sse2.pcmp") ||
2261 Name.startswith(
"avx2.pcmp"))) {
2263 bool CmpEq =
Name[9] ==
'e';
2264 Rep =
Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2267 }
else if (IsX86 && (
Name.startswith(
"avx512.broadcastm"))) {
2274 Rep =
Builder.CreateVectorSplat(NumElts, Rep);
2275 }
else if (IsX86 && (
Name ==
"sse.sqrt.ss" ||
2276 Name ==
"sse2.sqrt.sd")) {
2280 Intrinsic::sqrt, Elt0->
getType());
2283 }
else if (IsX86 && (
Name.startswith(
"avx.sqrt.p") ||
2284 Name.startswith(
"sse2.sqrt.p") ||
2285 Name.startswith(
"sse.sqrt.p"))) {
2289 {CI->getArgOperand(0)});
2290 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.sqrt.p"))) {
2293 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2295 : Intrinsic::x86_avx512_sqrt_pd_512;
2304 {CI->getArgOperand(0)});
2308 }
else if (IsX86 && (
Name.startswith(
"avx512.ptestm") ||
2309 Name.startswith(
"avx512.ptestnm"))) {
2313 Rep =
Builder.CreateAnd(Op0, Op1);
2317 Name.startswith(
"avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2318 Rep =
Builder.CreateICmp(Pred, Rep, Zero);
2320 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.pbroadcast"))){
2326 }
else if (IsX86 && (
Name.startswith(
"avx512.kunpck"))) {
2331 for (
unsigned i = 0; i != NumElts; ++i)
2344 }
else if (IsX86 &&
Name ==
"avx512.kand.w") {
2349 }
else if (IsX86 &&
Name ==
"avx512.kandn.w") {
2355 }
else if (IsX86 &&
Name ==
"avx512.kor.w") {
2360 }
else if (IsX86 &&
Name ==
"avx512.kxor.w") {
2365 }
else if (IsX86 &&
Name ==
"avx512.kxnor.w") {
2371 }
else if (IsX86 &&
Name ==
"avx512.knot.w") {
2376 (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w")) {
2382 if (
Name[14] ==
'c')
2383 C = ConstantInt::getAllOnesValue(
Builder.getInt16Ty());
2385 C = ConstantInt::getNullValue(
Builder.getInt16Ty());
2386 Rep =
Builder.CreateICmpEQ(Rep,
C);
2388 }
else if (IsX86 && (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2389 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2390 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2391 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd")) {
2398 if (
Name.contains(
".add."))
2399 EltOp =
Builder.CreateFAdd(Elt0, Elt1);
2400 else if (
Name.contains(
".sub."))
2401 EltOp =
Builder.CreateFSub(Elt0, Elt1);
2402 else if (
Name.contains(
".mul."))
2403 EltOp =
Builder.CreateFMul(Elt0, Elt1);
2405 EltOp =
Builder.CreateFDiv(Elt0, Elt1);
2408 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.pcmp")) {
2410 bool CmpEq =
Name[16] ==
'e';
2412 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.vpshufbitqmb.")) {
2418 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
break;
2419 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
break;
2420 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
break;
2424 { CI->getOperand(0), CI->getArgOperand(1) });
2426 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.fpclass.p")) {
2431 if (VecWidth == 128 && EltWidth == 32)
2432 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2433 else if (VecWidth == 256 && EltWidth == 32)
2434 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2435 else if (VecWidth == 512 && EltWidth == 32)
2436 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2437 else if (VecWidth == 128 && EltWidth == 64)
2438 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2439 else if (VecWidth == 256 && EltWidth == 64)
2440 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2441 else if (VecWidth == 512 && EltWidth == 64)
2442 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2447 { CI->getOperand(0), CI->getArgOperand(1) });
2449 }
else if (IsX86 &&
Name.startswith(
"avx512.cmp.p")) {
2451 Type *OpTy = Args[0]->getType();
2455 if (VecWidth == 128 && EltWidth == 32)
2456 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2457 else if (VecWidth == 256 && EltWidth == 32)
2458 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2459 else if (VecWidth == 512 && EltWidth == 32)
2460 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2461 else if (VecWidth == 128 && EltWidth == 64)
2462 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2463 else if (VecWidth == 256 && EltWidth == 64)
2464 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2465 else if (VecWidth == 512 && EltWidth == 64)
2466 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2471 if (VecWidth == 512)
2473 Args.push_back(Mask);
2477 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.cmp.")) {
2479 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2481 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.ucmp.")) {
2482 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2484 }
else if (IsX86 && (
Name.startswith(
"avx512.cvtb2mask.") ||
2485 Name.startswith(
"avx512.cvtw2mask.") ||
2486 Name.startswith(
"avx512.cvtd2mask.") ||
2487 Name.startswith(
"avx512.cvtq2mask."))) {
2490 Rep =
Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2492 }
else if(IsX86 && (
Name ==
"ssse3.pabs.b.128" ||
2493 Name ==
"ssse3.pabs.w.128" ||
2494 Name ==
"ssse3.pabs.d.128" ||
2495 Name.startswith(
"avx2.pabs") ||
2496 Name.startswith(
"avx512.mask.pabs"))) {
2498 }
else if (IsX86 && (
Name ==
"sse41.pmaxsb" ||
2499 Name ==
"sse2.pmaxs.w" ||
2500 Name ==
"sse41.pmaxsd" ||
2501 Name.startswith(
"avx2.pmaxs") ||
2502 Name.startswith(
"avx512.mask.pmaxs"))) {
2504 }
else if (IsX86 && (
Name ==
"sse2.pmaxu.b" ||
2505 Name ==
"sse41.pmaxuw" ||
2506 Name ==
"sse41.pmaxud" ||
2507 Name.startswith(
"avx2.pmaxu") ||
2508 Name.startswith(
"avx512.mask.pmaxu"))) {
2510 }
else if (IsX86 && (
Name ==
"sse41.pminsb" ||
2511 Name ==
"sse2.pmins.w" ||
2512 Name ==
"sse41.pminsd" ||
2513 Name.startswith(
"avx2.pmins") ||
2514 Name.startswith(
"avx512.mask.pmins"))) {
2516 }
else if (IsX86 && (
Name ==
"sse2.pminu.b" ||
2517 Name ==
"sse41.pminuw" ||
2518 Name ==
"sse41.pminud" ||
2519 Name.startswith(
"avx2.pminu") ||
2520 Name.startswith(
"avx512.mask.pminu"))) {
2522 }
else if (IsX86 && (
Name ==
"sse2.pmulu.dq" ||
2523 Name ==
"avx2.pmulu.dq" ||
2524 Name ==
"avx512.pmulu.dq.512" ||
2525 Name.startswith(
"avx512.mask.pmulu.dq."))) {
2527 }
else if (IsX86 && (
Name ==
"sse41.pmuldq" ||
2528 Name ==
"avx2.pmul.dq" ||
2529 Name ==
"avx512.pmul.dq.512" ||
2530 Name.startswith(
"avx512.mask.pmul.dq."))) {
2532 }
else if (IsX86 && (
Name ==
"sse.cvtsi2ss" ||
2533 Name ==
"sse2.cvtsi2sd" ||
2534 Name ==
"sse.cvtsi642ss" ||
2535 Name ==
"sse2.cvtsi642sd")) {
2538 cast<VectorType>(CI->
getType())->getElementType());
2540 }
else if (IsX86 &&
Name ==
"avx512.cvtusi2sd") {
2543 cast<VectorType>(CI->
getType())->getElementType());
2545 }
else if (IsX86 &&
Name ==
"sse2.cvtss2sd") {
2548 Rep, cast<VectorType>(CI->
getType())->getElementType());
2550 }
else if (IsX86 && (
Name ==
"sse2.cvtdq2pd" ||
2551 Name ==
"sse2.cvtdq2ps" ||
2552 Name ==
"avx.cvtdq2.pd.256" ||
2553 Name ==
"avx.cvtdq2.ps.256" ||
2554 Name.startswith(
"avx512.mask.cvtdq2pd.") ||
2555 Name.startswith(
"avx512.mask.cvtudq2pd.") ||
2556 Name.startswith(
"avx512.mask.cvtdq2ps.") ||
2557 Name.startswith(
"avx512.mask.cvtudq2ps.") ||
2558 Name.startswith(
"avx512.mask.cvtqq2pd.") ||
2559 Name.startswith(
"avx512.mask.cvtuqq2pd.") ||
2560 Name ==
"avx512.mask.cvtqq2ps.256" ||
2561 Name ==
"avx512.mask.cvtqq2ps.512" ||
2562 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2563 Name ==
"avx512.mask.cvtuqq2ps.512" ||
2564 Name ==
"sse2.cvtps2pd" ||
2565 Name ==
"avx.cvt.ps2.pd.256" ||
2566 Name ==
"avx512.mask.cvtps2pd.128" ||
2567 Name ==
"avx512.mask.cvtps2pd.256")) {
2568 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2570 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2572 unsigned NumDstElts = DstTy->getNumElements();
2573 if (NumDstElts < SrcTy->getNumElements()) {
2574 assert(NumDstElts == 2 &&
"Unexpected vector size");
2578 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2581 Rep =
Builder.CreateFPExt(Rep, DstTy,
"cvtps2pd");
2584 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2585 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2586 : Intrinsic::x86_avx512_sitofp_round;
2591 Rep = IsUnsigned ?
Builder.CreateUIToFP(Rep, DstTy,
"cvt")
2592 :
Builder.CreateSIToFP(Rep, DstTy,
"cvt");
2598 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vcvtph2ps.") ||
2599 Name.startswith(
"vcvtph2ps."))) {
2600 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2602 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2603 unsigned NumDstElts = DstTy->getNumElements();
2604 if (NumDstElts != SrcTy->getNumElements()) {
2605 assert(NumDstElts == 4 &&
"Unexpected vector size");
2610 Rep =
Builder.CreateFPExt(Rep, DstTy,
"cvtph2ps");
2614 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.load")) {
2620 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.expand.load.")) {
2621 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2622 Type *PtrTy = ResultTy->getElementType();
2629 ResultTy->getNumElements());
2632 Intrinsic::masked_expandload,
2635 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.compress.store.")) {
2637 Type *PtrTy = ResultTy->getElementType();
2645 cast<FixedVectorType>(ResultTy)->getNumElements());
2648 Intrinsic::masked_compressstore,
2651 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.compress.") ||
2652 Name.startswith(
"avx512.mask.expand."))) {
2653 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2656 ResultTy->getNumElements());
2658 bool IsCompress =
Name[12] ==
'c';
2659 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2660 : Intrinsic::x86_avx512_mask_expand;
2664 }
else if (IsX86 &&
Name.startswith(
"xop.vpcom")) {
2666 if (
Name.endswith(
"ub") ||
Name.endswith(
"uw") ||
Name.endswith(
"ud") ||
2667 Name.endswith(
"uq"))
2669 else if (
Name.endswith(
"b") ||
Name.endswith(
"w") ||
Name.endswith(
"d") ||
2677 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2680 if (
Name.startswith(
"lt"))
2682 else if (
Name.startswith(
"le"))
2684 else if (
Name.startswith(
"gt"))
2686 else if (
Name.startswith(
"ge"))
2688 else if (
Name.startswith(
"eq"))
2690 else if (
Name.startswith(
"ne"))
2692 else if (
Name.startswith(
"false"))
2694 else if (
Name.startswith(
"true"))
2701 }
else if (IsX86 &&
Name.startswith(
"xop.vpcmov")) {
2706 Rep =
Builder.CreateOr(Sel0, Sel1);
2707 }
else if (IsX86 && (
Name.startswith(
"xop.vprot") ||
2708 Name.startswith(
"avx512.prol") ||
2709 Name.startswith(
"avx512.mask.prol"))) {
2711 }
else if (IsX86 && (
Name.startswith(
"avx512.pror") ||
2712 Name.startswith(
"avx512.mask.pror"))) {
2714 }
else if (IsX86 && (
Name.startswith(
"avx512.vpshld.") ||
2715 Name.startswith(
"avx512.mask.vpshld") ||
2716 Name.startswith(
"avx512.maskz.vpshld"))) {
2717 bool ZeroMask =
Name[11] ==
'z';
2719 }
else if (IsX86 && (
Name.startswith(
"avx512.vpshrd.") ||
2720 Name.startswith(
"avx512.mask.vpshrd") ||
2721 Name.startswith(
"avx512.maskz.vpshrd"))) {
2722 bool ZeroMask =
Name[11] ==
'z';
2724 }
else if (IsX86 &&
Name ==
"sse42.crc32.64.8") {
2726 Intrinsic::x86_sse42_crc32_32_8);
2730 }
else if (IsX86 && (
Name.startswith(
"avx.vbroadcast.s") ||
2731 Name.startswith(
"avx512.vbroadcast.s"))) {
2733 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2734 Type *EltTy = VecTy->getElementType();
2735 unsigned EltNum = VecTy->getNumElements();
2741 for (
unsigned I = 0;
I < EltNum; ++
I)
2742 Rep =
Builder.CreateInsertElement(Rep, Load,
2744 }
else if (IsX86 && (
Name.startswith(
"sse41.pmovsx") ||
2745 Name.startswith(
"sse41.pmovzx") ||
2746 Name.startswith(
"avx2.pmovsx") ||
2747 Name.startswith(
"avx2.pmovzx") ||
2748 Name.startswith(
"avx512.mask.pmovsx") ||
2749 Name.startswith(
"avx512.mask.pmovzx"))) {
2750 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2751 unsigned NumDstElts = DstTy->getNumElements();
2755 for (
unsigned i = 0; i != NumDstElts; ++i)
2762 Rep = DoSext ?
Builder.CreateSExt(SV, DstTy)
2763 :
Builder.CreateZExt(SV, DstTy);
2768 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
2769 Name ==
"avx512.mask.pmov.qd.512" ||
2770 Name ==
"avx512.mask.pmov.wb.256" ||
2771 Name ==
"avx512.mask.pmov.wb.512") {
2776 }
else if (IsX86 && (
Name.startswith(
"avx.vbroadcastf128") ||
2777 Name ==
"avx2.vbroadcasti128")) {
2779 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
2783 PointerType::getUnqual(VT));
2785 if (NumSrcElts == 2)
2788 Rep =
Builder.CreateShuffleVector(
2790 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.shuf.i") ||
2791 Name.startswith(
"avx512.mask.shuf.f"))) {
2792 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2796 unsigned ControlBitsMask = NumLanes - 1;
2797 unsigned NumControlBits = NumLanes / 2;
2800 for (
unsigned l = 0; l != NumLanes; ++l) {
2801 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2803 if (l >= NumLanes / 2)
2804 LaneMask += NumLanes;
2805 for (
unsigned i = 0; i != NumElementsInLane; ++i)
2806 ShuffleMask.
push_back(LaneMask * NumElementsInLane + i);
2812 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.broadcastf") ||
2813 Name.startswith(
"avx512.mask.broadcasti"))) {
2814 unsigned NumSrcElts =
2817 unsigned NumDstElts =
2818 cast<FixedVectorType>(CI->
getType())->getNumElements();
2821 for (
unsigned i = 0; i != NumDstElts; ++i)
2822 ShuffleMask[i] = i % NumSrcElts;
2829 }
else if (IsX86 && (
Name.startswith(
"avx2.pbroadcast") ||
2830 Name.startswith(
"avx2.vbroadcast") ||
2831 Name.startswith(
"avx512.pbroadcast") ||
2832 Name.startswith(
"avx512.mask.broadcast.s"))) {
2839 Rep =
Builder.CreateShuffleVector(Op, M);
2844 }
else if (IsX86 && (
Name.startswith(
"sse2.padds.") ||
2845 Name.startswith(
"avx2.padds.") ||
2846 Name.startswith(
"avx512.padds.") ||
2847 Name.startswith(
"avx512.mask.padds."))) {
2849 }
else if (IsX86 && (
Name.startswith(
"sse2.psubs.") ||
2850 Name.startswith(
"avx2.psubs.") ||
2851 Name.startswith(
"avx512.psubs.") ||
2852 Name.startswith(
"avx512.mask.psubs."))) {
2854 }
else if (IsX86 && (
Name.startswith(
"sse2.paddus.") ||
2855 Name.startswith(
"avx2.paddus.") ||
2856 Name.startswith(
"avx512.mask.paddus."))) {
2858 }
else if (IsX86 && (
Name.startswith(
"sse2.psubus.") ||
2859 Name.startswith(
"avx2.psubus.") ||
2860 Name.startswith(
"avx512.mask.psubus."))) {
2862 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.palignr.")) {
2869 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.valign.")) {
2876 }
else if (IsX86 && (
Name ==
"sse2.psll.dq" ||
2877 Name ==
"avx2.psll.dq")) {
2879 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2882 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq" ||
2883 Name ==
"avx2.psrl.dq")) {
2885 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2888 }
else if (IsX86 && (
Name ==
"sse2.psll.dq.bs" ||
2889 Name ==
"avx2.psll.dq.bs" ||
2890 Name ==
"avx512.psll.dq.512")) {
2892 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2894 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq.bs" ||
2895 Name ==
"avx2.psrl.dq.bs" ||
2896 Name ==
"avx512.psrl.dq.512")) {
2898 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2900 }
else if (IsX86 && (
Name ==
"sse41.pblendw" ||
2901 Name.startswith(
"sse41.blendp") ||
2902 Name.startswith(
"avx.blend.p") ||
2903 Name ==
"avx2.pblendw" ||
2904 Name.startswith(
"avx2.pblendd."))) {
2907 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2908 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2909 unsigned NumElts = VecTy->getNumElements();
2912 for (
unsigned i = 0; i != NumElts; ++i)
2913 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2915 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
2916 }
else if (IsX86 && (
Name.startswith(
"avx.vinsertf128.") ||
2917 Name ==
"avx2.vinserti128" ||
2918 Name.startswith(
"avx512.mask.insert"))) {
2921 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2922 unsigned DstNumElts =
2923 cast<FixedVectorType>(CI->
getType())->getNumElements();
2924 unsigned SrcNumElts =
2925 cast<FixedVectorType>(Op1->
getType())->getNumElements();
2926 unsigned Scale = DstNumElts / SrcNumElts;
2933 for (
unsigned i = 0; i != SrcNumElts; ++i)
2935 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
2936 Idxs[i] = SrcNumElts;
2937 Rep =
Builder.CreateShuffleVector(Op1, Idxs);
2951 for (
unsigned i = 0; i != DstNumElts; ++i)
2954 for (
unsigned i = 0; i != SrcNumElts; ++i)
2955 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2956 Rep =
Builder.CreateShuffleVector(Op0, Rep, Idxs);
2962 }
else if (IsX86 && (
Name.startswith(
"avx.vextractf128.") ||
2963 Name ==
"avx2.vextracti128" ||
2964 Name.startswith(
"avx512.mask.vextract"))) {
2966 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2967 unsigned DstNumElts =
2968 cast<FixedVectorType>(CI->
getType())->getNumElements();
2969 unsigned SrcNumElts =
2970 cast<FixedVectorType>(Op0->
getType())->getNumElements();
2971 unsigned Scale = SrcNumElts / DstNumElts;
2978 for (
unsigned i = 0; i != DstNumElts; ++i) {
2979 Idxs[i] = i + (Imm * DstNumElts);
2981 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2987 }
else if (!IsX86 &&
Name ==
"stackprotectorcheck") {
2989 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.perm.df.") ||
2990 Name.startswith(
"avx512.mask.perm.di."))) {
2992 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2993 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2994 unsigned NumElts = VecTy->getNumElements();
2997 for (
unsigned i = 0; i != NumElts; ++i)
2998 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3000 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
3005 }
else if (IsX86 && (
Name.startswith(
"avx.vperm2f128.") ||
3006 Name ==
"avx2.vperm2i128")) {
3015 uint8_t Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3017 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3018 unsigned HalfSize = NumElts / 2;
3030 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3031 for (
unsigned i = 0; i < HalfSize; ++i)
3032 ShuffleMask[i] = StartIndex + i;
3035 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3036 for (
unsigned i = 0; i < HalfSize; ++i)
3037 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3039 Rep =
Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3041 }
else if (IsX86 && (
Name.startswith(
"avx.vpermil.") ||
3042 Name ==
"sse2.pshuf.d" ||
3043 Name.startswith(
"avx512.mask.vpermil.p") ||
3044 Name.startswith(
"avx512.mask.pshuf.d."))) {
3046 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3047 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3048 unsigned NumElts = VecTy->getNumElements();
3050 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3051 unsigned IdxMask = ((1 << IdxSize) - 1);
3057 for (
unsigned i = 0; i != NumElts; ++i)
3058 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3060 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
3065 }
else if (IsX86 && (
Name ==
"sse2.pshufl.w" ||
3066 Name.startswith(
"avx512.mask.pshufl.w."))) {
3068 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3069 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3072 for (
unsigned l = 0; l != NumElts; l += 8) {
3073 for (
unsigned i = 0; i != 4; ++i)
3074 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3075 for (
unsigned i = 4; i != 8; ++i)
3076 Idxs[i + l] = i + l;
3079 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
3084 }
else if (IsX86 && (
Name ==
"sse2.pshufh.w" ||
3085 Name.startswith(
"avx512.mask.pshufh.w."))) {
3087 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3088 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3091 for (
unsigned l = 0; l != NumElts; l += 8) {
3092 for (
unsigned i = 0; i != 4; ++i)
3093 Idxs[i + l] = i + l;
3094 for (
unsigned i = 0; i != 4; ++i)
3095 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3098 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
3103 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.shuf.p")) {
3106 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3107 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3110 unsigned HalfLaneElts = NumLaneElts / 2;
3113 for (
unsigned i = 0; i != NumElts; ++i) {
3115 Idxs[i] = i - (i % NumLaneElts);
3117 if ((i % NumLaneElts) >= HalfLaneElts)
3121 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3124 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
3128 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.movddup") ||
3129 Name.startswith(
"avx512.mask.movshdup") ||
3130 Name.startswith(
"avx512.mask.movsldup"))) {
3132 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3136 if (
Name.startswith(
"avx512.mask.movshdup."))
3140 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
3141 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
3142 Idxs[i + l + 0] = i + l +
Offset;
3143 Idxs[i + l + 1] = i + l +
Offset;
3146 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
3150 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.punpckl") ||
3151 Name.startswith(
"avx512.mask.unpckl."))) {
3154 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3158 for (
int l = 0; l != NumElts; l += NumLaneElts)
3159 for (
int i = 0; i != NumLaneElts; ++i)
3160 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3162 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
3166 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.punpckh") ||
3167 Name.startswith(
"avx512.mask.unpckh."))) {
3170 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3174 for (
int l = 0; l != NumElts; l += NumLaneElts)
3175 for (
int i = 0; i != NumLaneElts; ++i)
3176 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3178 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
3182 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.and.") ||
3183 Name.startswith(
"avx512.mask.pand."))) {
3185 VectorType *ITy = VectorType::getInteger(FTy);
3188 Rep =
Builder.CreateBitCast(Rep, FTy);
3191 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.andn.") ||
3192 Name.startswith(
"avx512.mask.pandn."))) {
3194 VectorType *ITy = VectorType::getInteger(FTy);
3198 Rep =
Builder.CreateBitCast(Rep, FTy);
3201 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.or.") ||
3202 Name.startswith(
"avx512.mask.por."))) {
3204 VectorType *ITy = VectorType::getInteger(FTy);
3207 Rep =
Builder.CreateBitCast(Rep, FTy);
3210 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.xor.") ||
3211 Name.startswith(
"avx512.mask.pxor."))) {
3213 VectorType *ITy = VectorType::getInteger(FTy);
3216 Rep =
Builder.CreateBitCast(Rep, FTy);
3219 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.padd.")) {
3223 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psub.")) {
3227 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.pmull.")) {
3231 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.add.p")) {
3232 if (
Name.endswith(
".512")) {
3234 if (
Name[17] ==
's')
3235 IID = Intrinsic::x86_avx512_add_ps_512;
3237 IID = Intrinsic::x86_avx512_add_pd_512;
3240 { CI->getArgOperand(0), CI->getArgOperand(1),
3241 CI->getArgOperand(4) });
3247 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.div.p")) {
3248 if (
Name.endswith(
".512")) {
3250 if (
Name[17] ==
's')
3251 IID = Intrinsic::x86_avx512_div_ps_512;
3253 IID = Intrinsic::x86_avx512_div_pd_512;
3256 { CI->getArgOperand(0), CI->getArgOperand(1),
3257 CI->getArgOperand(4) });
3263 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.mul.p")) {
3264 if (
Name.endswith(
".512")) {
3266 if (
Name[17] ==
's')
3267 IID = Intrinsic::x86_avx512_mul_ps_512;
3269 IID = Intrinsic::x86_avx512_mul_pd_512;
3272 { CI->getArgOperand(0), CI->getArgOperand(1),
3273 CI->getArgOperand(4) });
3279 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.sub.p")) {
3280 if (
Name.endswith(
".512")) {
3282 if (
Name[17] ==
's')
3283 IID = Intrinsic::x86_avx512_sub_ps_512;
3285 IID = Intrinsic::x86_avx512_sub_pd_512;
3288 { CI->getArgOperand(0), CI->getArgOperand(1),
3289 CI->getArgOperand(4) });
3295 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.max.p") ||
3296 Name.startswith(
"avx512.mask.min.p")) &&
3297 Name.drop_front(18) ==
".512") {
3298 bool IsDouble =
Name[17] ==
'd';
3299 bool IsMin =
Name[13] ==
'i';
3301 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3302 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3307 { CI->getArgOperand(0), CI->getArgOperand(1),
3308 CI->getArgOperand(4) });
3311 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.lzcnt.")) {
3315 { CI->getArgOperand(0), Builder.getInt1(false) });
3318 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psll")) {
3319 bool IsImmediate =
Name[16] ==
'i' ||
3320 (
Name.size() > 18 &&
Name[18] ==
'i');
3321 bool IsVariable =
Name[16] ==
'v';
3328 if (IsVariable &&
Name[17] !=
'.') {
3329 if (
Size ==
'd' &&
Name[17] ==
'2')
3330 IID = Intrinsic::x86_avx2_psllv_q;
3331 else if (
Size ==
'd' &&
Name[17] ==
'4')
3332 IID = Intrinsic::x86_avx2_psllv_q_256;
3333 else if (
Size ==
's' &&
Name[17] ==
'4')
3334 IID = Intrinsic::x86_avx2_psllv_d;
3335 else if (
Size ==
's' &&
Name[17] ==
'8')
3336 IID = Intrinsic::x86_avx2_psllv_d_256;
3337 else if (
Size ==
'h' &&
Name[17] ==
'8')
3338 IID = Intrinsic::x86_avx512_psllv_w_128;
3339 else if (
Size ==
'h' &&
Name[17] ==
'1')
3340 IID = Intrinsic::x86_avx512_psllv_w_256;
3341 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3342 IID = Intrinsic::x86_avx512_psllv_w_512;
3345 }
else if (
Name.endswith(
".128")) {
3347 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3348 : Intrinsic::x86_sse2_psll_d;
3349 else if (
Size ==
'q')
3350 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3351 : Intrinsic::x86_sse2_psll_q;
3352 else if (
Size ==
'w')
3353 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3354 : Intrinsic::x86_sse2_psll_w;
3357 }
else if (
Name.endswith(
".256")) {
3359 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3360 : Intrinsic::x86_avx2_psll_d;
3361 else if (
Size ==
'q')
3362 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3363 : Intrinsic::x86_avx2_psll_q;
3364 else if (
Size ==
'w')
3365 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3366 : Intrinsic::x86_avx2_psll_w;
3371 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3372 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3373 Intrinsic::x86_avx512_psll_d_512;
3374 else if (
Size ==
'q')
3375 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3376 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3377 Intrinsic::x86_avx512_psll_q_512;
3378 else if (
Size ==
'w')
3379 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3380 : Intrinsic::x86_avx512_psll_w_512;
3386 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psrl")) {
3387 bool IsImmediate =
Name[16] ==
'i' ||
3388 (
Name.size() > 18 &&
Name[18] ==
'i');
3389 bool IsVariable =
Name[16] ==
'v';
3396 if (IsVariable &&
Name[17] !=
'.') {
3397 if (
Size ==
'd' &&
Name[17] ==
'2')
3398 IID = Intrinsic::x86_avx2_psrlv_q;
3399 else if (
Size ==
'd' &&
Name[17] ==
'4')
3400 IID = Intrinsic::x86_avx2_psrlv_q_256;
3401 else if (
Size ==
's' &&
Name[17] ==
'4')
3402 IID = Intrinsic::x86_avx2_psrlv_d;
3403 else if (
Size ==
's' &&
Name[17] ==
'8')
3404 IID = Intrinsic::x86_avx2_psrlv_d_256;
3405 else if (
Size ==
'h' &&
Name[17] ==
'8')
3406 IID = Intrinsic::x86_avx512_psrlv_w_128;
3407 else if (
Size ==
'h' &&
Name[17] ==
'1')
3408 IID = Intrinsic::x86_avx512_psrlv_w_256;
3409 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3410 IID = Intrinsic::x86_avx512_psrlv_w_512;
3413 }
else if (
Name.endswith(
".128")) {
3415 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3416 : Intrinsic::x86_sse2_psrl_d;
3417 else if (
Size ==
'q')
3418 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3419 : Intrinsic::x86_sse2_psrl_q;
3420 else if (
Size ==
'w')
3421 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3422 : Intrinsic::x86_sse2_psrl_w;
3425 }
else if (
Name.endswith(
".256")) {
3427 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3428 : Intrinsic::x86_avx2_psrl_d;
3429 else if (
Size ==
'q')
3430 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3431 : Intrinsic::x86_avx2_psrl_q;
3432 else if (
Size ==
'w')
3433 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3434 : Intrinsic::x86_avx2_psrl_w;
3439 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3440 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3441 Intrinsic::x86_avx512_psrl_d_512;
3442 else if (
Size ==
'q')
3443 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3444 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3445 Intrinsic::x86_avx512_psrl_q_512;
3446 else if (
Size ==
'w')
3447 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3448 : Intrinsic::x86_avx512_psrl_w_512;
3454 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psra")) {
3455 bool IsImmediate =
Name[16] ==
'i' ||
3456 (
Name.size() > 18 &&
Name[18] ==
'i');
3457 bool IsVariable =
Name[16] ==
'v';
3464 if (IsVariable &&
Name[17] !=
'.') {
3465 if (
Size ==
's' &&
Name[17] ==
'4')
3466 IID = Intrinsic::x86_avx2_psrav_d;
3467 else if (
Size ==
's' &&
Name[17] ==
'8')
3468 IID = Intrinsic::x86_avx2_psrav_d_256;
3469 else if (
Size ==
'h' &&
Name[17] ==
'8')
3470 IID = Intrinsic::x86_avx512_psrav_w_128;
3471 else if (
Size ==
'h' &&
Name[17] ==
'1')
3472 IID = Intrinsic::x86_avx512_psrav_w_256;
3473 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3474 IID = Intrinsic::x86_avx512_psrav_w_512;
3477 }
else if (
Name.endswith(
".128")) {
3479 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3480 : Intrinsic::x86_sse2_psra_d;
3481 else if (
Size ==
'q')
3482 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3483 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3484 Intrinsic::x86_avx512_psra_q_128;
3485 else if (
Size ==
'w')
3486 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3487 : Intrinsic::x86_sse2_psra_w;
3490 }
else if (
Name.endswith(
".256")) {
3492 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3493 : Intrinsic::x86_avx2_psra_d;
3494 else if (
Size ==
'q')
3495 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3496 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3497 Intrinsic::x86_avx512_psra_q_256;
3498 else if (
Size ==
'w')
3499 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3500 : Intrinsic::x86_avx2_psra_w;
3505 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3506 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3507 Intrinsic::x86_avx512_psra_d_512;
3508 else if (
Size ==
'q')
3509 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3510 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3511 Intrinsic::x86_avx512_psra_q_512;
3512 else if (
Size ==
'w')
3513 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3514 : Intrinsic::x86_avx512_psra_w_512;
3520 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.move.s")) {
3522 }
else if (IsX86 &&
Name.startswith(
"avx512.cvtmask2")) {
3524 }
else if (IsX86 &&
Name.endswith(
".movntdqa")) {
3533 Ptr, PointerType::getUnqual(CI->
getType()),
"cast");
3537 LI->
setMetadata(M->getMDKindID(
"nontemporal"), Node);
3539 }
else if (IsX86 && (
Name.startswith(
"fma.vfmadd.") ||
3540 Name.startswith(
"fma.vfmsub.") ||
3541 Name.startswith(
"fma.vfnmadd.") ||
3542 Name.startswith(
"fma.vfnmsub."))) {
3543 bool NegMul =
Name[6] ==
'n';
3544 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3545 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3556 if (NegMul && !IsScalar)
3557 Ops[0] =
Builder.CreateFNeg(Ops[0]);
3558 if (NegMul && IsScalar)
3559 Ops[1] =
Builder.CreateFNeg(Ops[1]);
3561 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3571 }
else if (IsX86 &&
Name.startswith(
"fma4.vfmadd.s")) {
3586 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmadd.s") ||
3587 Name.startswith(
"avx512.maskz.vfmadd.s") ||
3588 Name.startswith(
"avx512.mask3.vfmadd.s") ||
3589 Name.startswith(
"avx512.mask3.vfmsub.s") ||
3590 Name.startswith(
"avx512.mask3.vfnmsub.s"))) {
3591 bool IsMask3 =
Name[11] ==
'3';
3592 bool IsMaskZ =
Name[11] ==
'z';
3594 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3595 bool NegMul =
Name[2] ==
'n';
3596 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3602 if (NegMul && (IsMask3 || IsMaskZ))
3604 if (NegMul && !(IsMask3 || IsMaskZ))
3614 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3618 if (
Name.back() ==
'd')
3619 IID = Intrinsic::x86_avx512_vfmadd_f64;
3621 IID = Intrinsic::x86_avx512_vfmadd_f32;
3623 Rep =
Builder.CreateCall(FMA, Ops);
3636 if (NegAcc && IsMask3)
3644 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmadd.p") ||
3645 Name.startswith(
"avx512.mask.vfnmadd.p") ||
3646 Name.startswith(
"avx512.mask.vfnmsub.p") ||
3647 Name.startswith(
"avx512.mask3.vfmadd.p") ||
3648 Name.startswith(
"avx512.mask3.vfmsub.p") ||
3649 Name.startswith(
"avx512.mask3.vfnmsub.p") ||
3650 Name.startswith(
"avx512.maskz.vfmadd.p"))) {
3651 bool IsMask3 =
Name[11] ==
'3';
3652 bool IsMaskZ =
Name[11] ==
'z';
3654 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3655 bool NegMul =
Name[2] ==
'n';
3656 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3662 if (NegMul && (IsMask3 || IsMaskZ))
3664 if (NegMul && !(IsMask3 || IsMaskZ))
3671 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
3675 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3677 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3680 { A, B, C, CI->getArgOperand(4) });
3693 }
else if (IsX86 &&
Name.startswith(
"fma.vfmsubadd.p")) {
3697 if (VecWidth == 128 && EltWidth == 32)
3698 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3699 else if (VecWidth == 256 && EltWidth == 32)
3700 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3701 else if (VecWidth == 128 && EltWidth == 64)
3702 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3703 else if (VecWidth == 256 && EltWidth == 64)
3704 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3710 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3713 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmaddsub.p") ||
3714 Name.startswith(
"avx512.mask3.vfmaddsub.p") ||
3715 Name.startswith(
"avx512.maskz.vfmaddsub.p") ||
3716 Name.startswith(
"avx512.mask3.vfmsubadd.p"))) {
3717 bool IsMask3 =
Name[11] ==
'3';
3718 bool IsMaskZ =
Name[11] ==
'z';
3720 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3721 bool IsSubAdd =
Name[3] ==
's';
3726 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3728 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3733 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3738 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3746 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3753 for (
int i = 0; i != NumElts; ++i)
3754 Idxs[i] = i + (i % 2) * NumElts;
3756 Rep =
Builder.CreateShuffleVector(Even, Odd, Idxs);
3764 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.pternlog.") ||
3765 Name.startswith(
"avx512.maskz.pternlog."))) {
3766 bool ZeroMask =
Name[11] ==
'z';
3770 if (VecWidth == 128 && EltWidth == 32)
3771 IID = Intrinsic::x86_avx512_pternlog_d_128;
3772 else if (VecWidth == 256 && EltWidth == 32)
3773 IID = Intrinsic::x86_avx512_pternlog_d_256;
3774 else if (VecWidth == 512 && EltWidth == 32)
3775 IID = Intrinsic::x86_avx512_pternlog_d_512;
3776 else if (VecWidth == 128 && EltWidth == 64)
3777 IID = Intrinsic::x86_avx512_pternlog_q_128;
3778 else if (VecWidth == 256 && EltWidth == 64)
3779 IID = Intrinsic::x86_avx512_pternlog_q_256;
3780 else if (VecWidth == 512 && EltWidth == 64)
3781 IID = Intrinsic::x86_avx512_pternlog_q_512;
3792 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpmadd52") ||
3793 Name.startswith(
"avx512.maskz.vpmadd52"))) {
3794 bool ZeroMask =
Name[11] ==
'z';
3798 if (VecWidth == 128 && !
High)
3799 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3800 else if (VecWidth == 256 && !
High)
3801 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3802 else if (VecWidth == 512 && !
High)
3803 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3804 else if (VecWidth == 128 &&
High)
3805 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3806 else if (VecWidth == 256 &&
High)
3807 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3808 else if (VecWidth == 512 &&
High)
3809 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3820 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpermi2var.") ||
3821 Name.startswith(
"avx512.mask.vpermt2var.") ||
3822 Name.startswith(
"avx512.maskz.vpermt2var."))) {
3823 bool ZeroMask =
Name[11] ==
'z';
3824 bool IndexForm =
Name[17] ==
'i';
3826 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpdpbusd.") ||
3827 Name.startswith(
"avx512.maskz.vpdpbusd.") ||
3828 Name.startswith(
"avx512.mask.vpdpbusds.") ||
3829 Name.startswith(
"avx512.maskz.vpdpbusds."))) {
3830 bool ZeroMask =
Name[11] ==
'z';
3831 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3834 if (VecWidth == 128 && !IsSaturating)
3835 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3836 else if (VecWidth == 256 && !IsSaturating)
3837 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3838 else if (VecWidth == 512 && !IsSaturating)
3839 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3840 else if (VecWidth == 128 && IsSaturating)
3841 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3842 else if (VecWidth == 256 && IsSaturating)
3843 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3844 else if (VecWidth == 512 && IsSaturating)
3845 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3856 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpdpwssd.") ||
3857 Name.startswith(
"avx512.maskz.vpdpwssd.") ||
3858 Name.startswith(
"avx512.mask.vpdpwssds.") ||
3859 Name.startswith(
"avx512.maskz.vpdpwssds."))) {
3860 bool ZeroMask =
Name[11] ==
'z';
3861 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3864 if (VecWidth == 128 && !IsSaturating)
3865 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3866 else if (VecWidth == 256 && !IsSaturating)
3867 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3868 else if (VecWidth == 512 && !IsSaturating)
3869 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3870 else if (VecWidth == 128 && IsSaturating)
3871 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3872 else if (VecWidth == 256 && IsSaturating)
3873 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3874 else if (VecWidth == 512 && IsSaturating)
3875 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3886 }
else if (IsX86 && (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
3887 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
3888 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64")) {
3890 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
3891 IID = Intrinsic::x86_addcarry_32;
3892 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
3893 IID = Intrinsic::x86_addcarry_64;
3894 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
3895 IID = Intrinsic::x86_subborrow_32;
3896 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
3897 IID = Intrinsic::x86_subborrow_64;
3919 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.") &&
3922 }
else if (IsNVVM && (
Name ==
"abs.i" ||
Name ==
"abs.ll")) {
3927 Rep =
Builder.CreateSelect(Cmp,
Arg, Neg,
"abs");
3928 }
else if (IsNVVM && (
Name.startswith(
"atomic.load.add.f32.p") ||
3929 Name.startswith(
"atomic.load.add.f64.p"))) {
3933 AtomicOrdering::SequentiallyConsistent);
3934 }
else if (IsNVVM && (
Name ==
"max.i" ||
Name ==
"max.ll" ||
3935 Name ==
"max.ui" ||
Name ==
"max.ull")) {
3939 ?
Builder.CreateICmpUGE(Arg0, Arg1,
"max.cond")
3940 :
Builder.CreateICmpSGE(Arg0, Arg1,
"max.cond");
3941 Rep =
Builder.CreateSelect(Cmp, Arg0, Arg1,
"max");
3942 }
else if (IsNVVM && (
Name ==
"min.i" ||
Name ==
"min.ll" ||
3943 Name ==
"min.ui" ||
Name ==
"min.ull")) {
3947 ?
Builder.CreateICmpULE(Arg0, Arg1,
"min.cond")
3948 :
Builder.CreateICmpSLE(Arg0, Arg1,
"min.cond");
3949 Rep =
Builder.CreateSelect(Cmp, Arg0, Arg1,
"min");
3950 }
else if (IsNVVM &&
Name ==
"clz.ll") {
3956 {Arg, Builder.getFalse()},
"ctlz");
3957 Rep =
Builder.CreateTrunc(Ctlz,
Builder.getInt32Ty(),
"ctlz.trunc");
3958 }
else if (IsNVVM &&
Name ==
"popc.ll") {
3966 Rep =
Builder.CreateTrunc(Popc,
Builder.getInt32Ty(),
"ctpop.trunc");
3967 }
else if (IsNVVM &&
Name ==
"h2f") {
3969 F->getParent(), Intrinsic::convert_from_fp16,
3970 {Builder.getFloatTy()}),
3984 const auto &DefaultCase = [&]() ->
void {
3989 "Unknown function for CallBase upgrade and isn't just a name change");
3995 if (
auto *OldST = dyn_cast<StructType>(CI->
getType())) {
3997 "Return type must have changed");
3998 assert(OldST->getNumElements() ==
3999 cast<StructType>(NewFn->
getReturnType())->getNumElements() &&
4000 "Must have same number of elements");
4005 for (
unsigned Idx = 0;
Idx < OldST->getNumElements(); ++
Idx) {
4007 Res =
Builder.CreateInsertValue(Res, Elem,
Idx);
4026 case Intrinsic::arm_neon_vst1:
4027 case Intrinsic::arm_neon_vst2:
4028 case Intrinsic::arm_neon_vst3:
4029 case Intrinsic::arm_neon_vst4:
4030 case Intrinsic::arm_neon_vst2lane:
4031 case Intrinsic::arm_neon_vst3lane:
4032 case Intrinsic::arm_neon_vst4lane: {
4034 NewCall =
Builder.CreateCall(NewFn, Args);
4037 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4038 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4039 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4043 cast<ConstantInt>(Args[3])->getZExtValue());
4044 NewCall =
Builder.CreateCall(NewFn, Args);
4047 case Intrinsic::aarch64_sve_ld3_sret:
4048 case Intrinsic::aarch64_sve_ld4_sret:
4049 case Intrinsic::aarch64_sve_ld2_sret: {
4058 dyn_cast<ScalableVectorType>(
F->getReturnType());
4059 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4063 for (
unsigned I = 0;
I <
N;
I++) {
4068 NewCall = dyn_cast<CallInst>(Ret);
4072 case Intrinsic::vector_extract: {
4075 if (!
Name.startswith(
"aarch64.sve.tuple.get")) {
4080 dyn_cast<ScalableVectorType>(
F->getReturnType());
4081 unsigned MinElts =
RetTy->getMinNumElements();
4082 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4088 case Intrinsic::vector_insert: {
4091 if (!
Name.startswith(
"aarch64.sve.tuple")) {
4095 if (
Name.startswith(
"aarch64.sve.tuple.set")) {
4096 unsigned I = dyn_cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4105 if (
Name.startswith(
"aarch64.sve.tuple.create")) {