30#include "llvm/IR/IntrinsicsAArch64.h"
31#include "llvm/IR/IntrinsicsARM.h"
32#include "llvm/IR/IntrinsicsNVPTX.h"
33#include "llvm/IR/IntrinsicsRISCV.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/IR/IntrinsicsX86.h"
50 cl::desc(
"Disable autoupgrade of debug info"));
60 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
75 Type *LastArgType =
F->getFunctionType()->getParamType(
76 F->getFunctionType()->getNumParams() - 1);
91 if (
F->getReturnType()->isVectorTy())
101 if (
F->getReturnType()->getScalarType()->isBFloatTy())
111 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
125 if (
Name.consume_front(
"avx."))
126 return (
Name.starts_with(
"blend.p") ||
127 Name ==
"cvt.ps2.pd.256" ||
128 Name ==
"cvtdq2.pd.256" ||
129 Name ==
"cvtdq2.ps.256" ||
130 Name.starts_with(
"movnt.") ||
131 Name.starts_with(
"sqrt.p") ||
132 Name.starts_with(
"storeu.") ||
133 Name.starts_with(
"vbroadcast.s") ||
134 Name.starts_with(
"vbroadcastf128") ||
135 Name.starts_with(
"vextractf128.") ||
136 Name.starts_with(
"vinsertf128.") ||
137 Name.starts_with(
"vperm2f128.") ||
138 Name.starts_with(
"vpermil."));
140 if (
Name.consume_front(
"avx2."))
141 return (
Name ==
"movntdqa" ||
142 Name.starts_with(
"pabs.") ||
143 Name.starts_with(
"padds.") ||
144 Name.starts_with(
"paddus.") ||
145 Name.starts_with(
"pblendd.") ||
147 Name.starts_with(
"pbroadcast") ||
148 Name.starts_with(
"pcmpeq.") ||
149 Name.starts_with(
"pcmpgt.") ||
150 Name.starts_with(
"pmax") ||
151 Name.starts_with(
"pmin") ||
152 Name.starts_with(
"pmovsx") ||
153 Name.starts_with(
"pmovzx") ||
155 Name ==
"pmulu.dq" ||
156 Name.starts_with(
"psll.dq") ||
157 Name.starts_with(
"psrl.dq") ||
158 Name.starts_with(
"psubs.") ||
159 Name.starts_with(
"psubus.") ||
160 Name.starts_with(
"vbroadcast") ||
161 Name ==
"vbroadcasti128" ||
162 Name ==
"vextracti128" ||
163 Name ==
"vinserti128" ||
164 Name ==
"vperm2i128");
166 if (
Name.consume_front(
"avx512.")) {
167 if (
Name.consume_front(
"mask."))
169 return (
Name.starts_with(
"add.p") ||
170 Name.starts_with(
"and.") ||
171 Name.starts_with(
"andn.") ||
172 Name.starts_with(
"broadcast.s") ||
173 Name.starts_with(
"broadcastf32x4.") ||
174 Name.starts_with(
"broadcastf32x8.") ||
175 Name.starts_with(
"broadcastf64x2.") ||
176 Name.starts_with(
"broadcastf64x4.") ||
177 Name.starts_with(
"broadcasti32x4.") ||
178 Name.starts_with(
"broadcasti32x8.") ||
179 Name.starts_with(
"broadcasti64x2.") ||
180 Name.starts_with(
"broadcasti64x4.") ||
181 Name.starts_with(
"cmp.b") ||
182 Name.starts_with(
"cmp.d") ||
183 Name.starts_with(
"cmp.q") ||
184 Name.starts_with(
"cmp.w") ||
185 Name.starts_with(
"compress.b") ||
186 Name.starts_with(
"compress.d") ||
187 Name.starts_with(
"compress.p") ||
188 Name.starts_with(
"compress.q") ||
189 Name.starts_with(
"compress.store.") ||
190 Name.starts_with(
"compress.w") ||
191 Name.starts_with(
"conflict.") ||
192 Name.starts_with(
"cvtdq2pd.") ||
193 Name.starts_with(
"cvtdq2ps.") ||
194 Name ==
"cvtpd2dq.256" ||
195 Name ==
"cvtpd2ps.256" ||
196 Name ==
"cvtps2pd.128" ||
197 Name ==
"cvtps2pd.256" ||
198 Name.starts_with(
"cvtqq2pd.") ||
199 Name ==
"cvtqq2ps.256" ||
200 Name ==
"cvtqq2ps.512" ||
201 Name ==
"cvttpd2dq.256" ||
202 Name ==
"cvttps2dq.128" ||
203 Name ==
"cvttps2dq.256" ||
204 Name.starts_with(
"cvtudq2pd.") ||
205 Name.starts_with(
"cvtudq2ps.") ||
206 Name.starts_with(
"cvtuqq2pd.") ||
207 Name ==
"cvtuqq2ps.256" ||
208 Name ==
"cvtuqq2ps.512" ||
209 Name.starts_with(
"dbpsadbw.") ||
210 Name.starts_with(
"div.p") ||
211 Name.starts_with(
"expand.b") ||
212 Name.starts_with(
"expand.d") ||
213 Name.starts_with(
"expand.load.") ||
214 Name.starts_with(
"expand.p") ||
215 Name.starts_with(
"expand.q") ||
216 Name.starts_with(
"expand.w") ||
217 Name.starts_with(
"fpclass.p") ||
218 Name.starts_with(
"insert") ||
219 Name.starts_with(
"load.") ||
220 Name.starts_with(
"loadu.") ||
221 Name.starts_with(
"lzcnt.") ||
222 Name.starts_with(
"max.p") ||
223 Name.starts_with(
"min.p") ||
224 Name.starts_with(
"movddup") ||
225 Name.starts_with(
"move.s") ||
226 Name.starts_with(
"movshdup") ||
227 Name.starts_with(
"movsldup") ||
228 Name.starts_with(
"mul.p") ||
229 Name.starts_with(
"or.") ||
230 Name.starts_with(
"pabs.") ||
231 Name.starts_with(
"packssdw.") ||
232 Name.starts_with(
"packsswb.") ||
233 Name.starts_with(
"packusdw.") ||
234 Name.starts_with(
"packuswb.") ||
235 Name.starts_with(
"padd.") ||
236 Name.starts_with(
"padds.") ||
237 Name.starts_with(
"paddus.") ||
238 Name.starts_with(
"palignr.") ||
239 Name.starts_with(
"pand.") ||
240 Name.starts_with(
"pandn.") ||
241 Name.starts_with(
"pavg") ||
242 Name.starts_with(
"pbroadcast") ||
243 Name.starts_with(
"pcmpeq.") ||
244 Name.starts_with(
"pcmpgt.") ||
245 Name.starts_with(
"perm.df.") ||
246 Name.starts_with(
"perm.di.") ||
247 Name.starts_with(
"permvar.") ||
248 Name.starts_with(
"pmaddubs.w.") ||
249 Name.starts_with(
"pmaddw.d.") ||
250 Name.starts_with(
"pmax") ||
251 Name.starts_with(
"pmin") ||
252 Name ==
"pmov.qd.256" ||
253 Name ==
"pmov.qd.512" ||
254 Name ==
"pmov.wb.256" ||
255 Name ==
"pmov.wb.512" ||
256 Name.starts_with(
"pmovsx") ||
257 Name.starts_with(
"pmovzx") ||
258 Name.starts_with(
"pmul.dq.") ||
259 Name.starts_with(
"pmul.hr.sw.") ||
260 Name.starts_with(
"pmulh.w.") ||
261 Name.starts_with(
"pmulhu.w.") ||
262 Name.starts_with(
"pmull.") ||
263 Name.starts_with(
"pmultishift.qb.") ||
264 Name.starts_with(
"pmulu.dq.") ||
265 Name.starts_with(
"por.") ||
266 Name.starts_with(
"prol.") ||
267 Name.starts_with(
"prolv.") ||
268 Name.starts_with(
"pror.") ||
269 Name.starts_with(
"prorv.") ||
270 Name.starts_with(
"pshuf.b.") ||
271 Name.starts_with(
"pshuf.d.") ||
272 Name.starts_with(
"pshufh.w.") ||
273 Name.starts_with(
"pshufl.w.") ||
274 Name.starts_with(
"psll.d") ||
275 Name.starts_with(
"psll.q") ||
276 Name.starts_with(
"psll.w") ||
277 Name.starts_with(
"pslli") ||
278 Name.starts_with(
"psllv") ||
279 Name.starts_with(
"psra.d") ||
280 Name.starts_with(
"psra.q") ||
281 Name.starts_with(
"psra.w") ||
282 Name.starts_with(
"psrai") ||
283 Name.starts_with(
"psrav") ||
284 Name.starts_with(
"psrl.d") ||
285 Name.starts_with(
"psrl.q") ||
286 Name.starts_with(
"psrl.w") ||
287 Name.starts_with(
"psrli") ||
288 Name.starts_with(
"psrlv") ||
289 Name.starts_with(
"psub.") ||
290 Name.starts_with(
"psubs.") ||
291 Name.starts_with(
"psubus.") ||
292 Name.starts_with(
"pternlog.") ||
293 Name.starts_with(
"punpckh") ||
294 Name.starts_with(
"punpckl") ||
295 Name.starts_with(
"pxor.") ||
296 Name.starts_with(
"shuf.f") ||
297 Name.starts_with(
"shuf.i") ||
298 Name.starts_with(
"shuf.p") ||
299 Name.starts_with(
"sqrt.p") ||
300 Name.starts_with(
"store.b.") ||
301 Name.starts_with(
"store.d.") ||
302 Name.starts_with(
"store.p") ||
303 Name.starts_with(
"store.q.") ||
304 Name.starts_with(
"store.w.") ||
305 Name ==
"store.ss" ||
306 Name.starts_with(
"storeu.") ||
307 Name.starts_with(
"sub.p") ||
308 Name.starts_with(
"ucmp.") ||
309 Name.starts_with(
"unpckh.") ||
310 Name.starts_with(
"unpckl.") ||
311 Name.starts_with(
"valign.") ||
312 Name ==
"vcvtph2ps.128" ||
313 Name ==
"vcvtph2ps.256" ||
314 Name.starts_with(
"vextract") ||
315 Name.starts_with(
"vfmadd.") ||
316 Name.starts_with(
"vfmaddsub.") ||
317 Name.starts_with(
"vfnmadd.") ||
318 Name.starts_with(
"vfnmsub.") ||
319 Name.starts_with(
"vpdpbusd.") ||
320 Name.starts_with(
"vpdpbusds.") ||
321 Name.starts_with(
"vpdpwssd.") ||
322 Name.starts_with(
"vpdpwssds.") ||
323 Name.starts_with(
"vpermi2var.") ||
324 Name.starts_with(
"vpermil.p") ||
325 Name.starts_with(
"vpermilvar.") ||
326 Name.starts_with(
"vpermt2var.") ||
327 Name.starts_with(
"vpmadd52") ||
328 Name.starts_with(
"vpshld.") ||
329 Name.starts_with(
"vpshldv.") ||
330 Name.starts_with(
"vpshrd.") ||
331 Name.starts_with(
"vpshrdv.") ||
332 Name.starts_with(
"vpshufbitqmb.") ||
333 Name.starts_with(
"xor."));
335 if (
Name.consume_front(
"mask3."))
337 return (
Name.starts_with(
"vfmadd.") ||
338 Name.starts_with(
"vfmaddsub.") ||
339 Name.starts_with(
"vfmsub.") ||
340 Name.starts_with(
"vfmsubadd.") ||
341 Name.starts_with(
"vfnmsub."));
343 if (
Name.consume_front(
"maskz."))
345 return (
Name.starts_with(
"pternlog.") ||
346 Name.starts_with(
"vfmadd.") ||
347 Name.starts_with(
"vfmaddsub.") ||
348 Name.starts_with(
"vpdpbusd.") ||
349 Name.starts_with(
"vpdpbusds.") ||
350 Name.starts_with(
"vpdpwssd.") ||
351 Name.starts_with(
"vpdpwssds.") ||
352 Name.starts_with(
"vpermt2var.") ||
353 Name.starts_with(
"vpmadd52") ||
354 Name.starts_with(
"vpshldv.") ||
355 Name.starts_with(
"vpshrdv."));
358 return (
Name ==
"movntdqa" ||
359 Name ==
"pmul.dq.512" ||
360 Name ==
"pmulu.dq.512" ||
361 Name.starts_with(
"broadcastm") ||
362 Name.starts_with(
"cmp.p") ||
363 Name.starts_with(
"cvtb2mask.") ||
364 Name.starts_with(
"cvtd2mask.") ||
365 Name.starts_with(
"cvtmask2") ||
366 Name.starts_with(
"cvtq2mask.") ||
367 Name ==
"cvtusi2sd" ||
368 Name.starts_with(
"cvtw2mask.") ||
373 Name ==
"kortestc.w" ||
374 Name ==
"kortestz.w" ||
375 Name.starts_with(
"kunpck") ||
378 Name.starts_with(
"padds.") ||
379 Name.starts_with(
"pbroadcast") ||
380 Name.starts_with(
"prol") ||
381 Name.starts_with(
"pror") ||
382 Name.starts_with(
"psll.dq") ||
383 Name.starts_with(
"psrl.dq") ||
384 Name.starts_with(
"psubs.") ||
385 Name.starts_with(
"ptestm") ||
386 Name.starts_with(
"ptestnm") ||
387 Name.starts_with(
"storent.") ||
388 Name.starts_with(
"vbroadcast.s") ||
389 Name.starts_with(
"vpshld.") ||
390 Name.starts_with(
"vpshrd."));
393 if (
Name.consume_front(
"fma."))
394 return (
Name.starts_with(
"vfmadd.") ||
395 Name.starts_with(
"vfmsub.") ||
396 Name.starts_with(
"vfmsubadd.") ||
397 Name.starts_with(
"vfnmadd.") ||
398 Name.starts_with(
"vfnmsub."));
400 if (
Name.consume_front(
"fma4."))
401 return Name.starts_with(
"vfmadd.s");
403 if (
Name.consume_front(
"sse."))
404 return (
Name ==
"add.ss" ||
405 Name ==
"cvtsi2ss" ||
406 Name ==
"cvtsi642ss" ||
409 Name.starts_with(
"sqrt.p") ||
411 Name.starts_with(
"storeu.") ||
414 if (
Name.consume_front(
"sse2."))
415 return (
Name ==
"add.sd" ||
416 Name ==
"cvtdq2pd" ||
417 Name ==
"cvtdq2ps" ||
418 Name ==
"cvtps2pd" ||
419 Name ==
"cvtsi2sd" ||
420 Name ==
"cvtsi642sd" ||
421 Name ==
"cvtss2sd" ||
424 Name.starts_with(
"padds.") ||
425 Name.starts_with(
"paddus.") ||
426 Name.starts_with(
"pcmpeq.") ||
427 Name.starts_with(
"pcmpgt.") ||
432 Name ==
"pmulu.dq" ||
433 Name.starts_with(
"pshuf") ||
434 Name.starts_with(
"psll.dq") ||
435 Name.starts_with(
"psrl.dq") ||
436 Name.starts_with(
"psubs.") ||
437 Name.starts_with(
"psubus.") ||
438 Name.starts_with(
"sqrt.p") ||
440 Name ==
"storel.dq" ||
441 Name.starts_with(
"storeu.") ||
444 if (
Name.consume_front(
"sse41."))
445 return (
Name.starts_with(
"blendp") ||
446 Name ==
"movntdqa" ||
456 Name.starts_with(
"pmovsx") ||
457 Name.starts_with(
"pmovzx") ||
460 if (
Name.consume_front(
"sse42."))
461 return Name ==
"crc32.64.8";
463 if (
Name.consume_front(
"sse4a."))
464 return Name.starts_with(
"movnt.");
466 if (
Name.consume_front(
"ssse3."))
467 return (
Name ==
"pabs.b.128" ||
468 Name ==
"pabs.d.128" ||
469 Name ==
"pabs.w.128");
471 if (
Name.consume_front(
"xop."))
472 return (
Name ==
"vpcmov" ||
473 Name ==
"vpcmov.256" ||
474 Name.starts_with(
"vpcom") ||
475 Name.starts_with(
"vprot"));
477 return (
Name ==
"addcarry.u32" ||
478 Name ==
"addcarry.u64" ||
479 Name ==
"addcarryx.u32" ||
480 Name ==
"addcarryx.u64" ||
481 Name ==
"subborrow.u32" ||
482 Name ==
"subborrow.u64" ||
483 Name.starts_with(
"vcvtph2ps."));
489 if (!
Name.consume_front(
"x86."))
497 if (
Name ==
"rdtscp") {
499 if (
F->getFunctionType()->getNumParams() == 0)
504 Intrinsic::x86_rdtscp);
511 if (
Name.consume_front(
"sse41.ptest")) {
513 .
Case(
"c", Intrinsic::x86_sse41_ptestc)
514 .
Case(
"z", Intrinsic::x86_sse41_ptestz)
515 .
Case(
"nzc", Intrinsic::x86_sse41_ptestnzc)
528 .
Case(
"sse41.insertps", Intrinsic::x86_sse41_insertps)
529 .
Case(
"sse41.dppd", Intrinsic::x86_sse41_dppd)
530 .
Case(
"sse41.dpps", Intrinsic::x86_sse41_dpps)
531 .
Case(
"sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
532 .
Case(
"avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
533 .
Case(
"avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
538 if (
Name.consume_front(
"avx512.mask.cmp.")) {
541 .
Case(
"pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
542 .
Case(
"pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
543 .
Case(
"pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
544 .
Case(
"ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
545 .
Case(
"ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
546 .
Case(
"ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
553 if (
Name.consume_front(
"avx512bf16.")) {
556 .
Case(
"cvtne2ps2bf16.128",
557 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
558 .
Case(
"cvtne2ps2bf16.256",
559 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
560 .
Case(
"cvtne2ps2bf16.512",
561 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
562 .
Case(
"mask.cvtneps2bf16.128",
563 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
564 .
Case(
"cvtneps2bf16.256",
565 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
566 .
Case(
"cvtneps2bf16.512",
567 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
574 .
Case(
"dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
575 .
Case(
"dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
576 .
Case(
"dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
583 if (
Name.consume_front(
"xop.")) {
585 if (
Name.starts_with(
"vpermil2")) {
588 auto Idx =
F->getFunctionType()->getParamType(2);
589 if (
Idx->isFPOrFPVectorTy()) {
590 unsigned IdxSize =
Idx->getPrimitiveSizeInBits();
591 unsigned EltSize =
Idx->getScalarSizeInBits();
592 if (EltSize == 64 && IdxSize == 128)
593 ID = Intrinsic::x86_xop_vpermil2pd;
594 else if (EltSize == 32 && IdxSize == 128)
595 ID = Intrinsic::x86_xop_vpermil2ps;
596 else if (EltSize == 64 && IdxSize == 256)
597 ID = Intrinsic::x86_xop_vpermil2pd_256;
599 ID = Intrinsic::x86_xop_vpermil2ps_256;
601 }
else if (
F->arg_size() == 2)
604 .
Case(
"vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
605 .
Case(
"vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
616 if (
Name ==
"seh.recoverfp") {
629 if (
Name.starts_with(
"rbit")) {
632 F->arg_begin()->getType());
636 if (
Name ==
"thread.pointer") {
643 bool Neon =
Name.consume_front(
"neon.");
648 if (
Name.consume_front(
"bfdot.")) {
651 .
Cases(
"v2f32.v8i8",
"v4f32.v16i8",
652 IsArm ? Intrinsic::arm_neon_bfdot
653 : Intrinsic::aarch64_neon_bfdot)
656 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
657 assert((OperandWidth == 64 || OperandWidth == 128) &&
658 "Unexpected operand width");
660 std::array<Type *, 2> Tys{
671 if (
Name.consume_front(
"bfm")) {
673 if (
Name.consume_back(
".v4f32.v16i8")) {
677 .
Case(
"mla", IsArm ? Intrinsic::arm_neon_bfmmla
678 : Intrinsic::aarch64_neon_bfmmla)
679 .
Case(
"lalb", IsArm ? Intrinsic::arm_neon_bfmlalb
680 : Intrinsic::aarch64_neon_bfmlalb)
681 .
Case(
"lalt", IsArm ? Intrinsic::arm_neon_bfmlalt
682 : Intrinsic::aarch64_neon_bfmlalt)
710 F->arg_begin()->getType());
714 if (
Name.consume_front(
"vst")) {
716 static const Regex vstRegex(
"^([1234]|[234]lane)\\.v[a-z0-9]*$");
720 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
721 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
724 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
725 Intrinsic::arm_neon_vst4lane};
727 auto fArgs =
F->getFunctionType()->params();
728 Type *Tys[] = {fArgs[0], fArgs[1]};
731 StoreInts[fArgs.size() - 3], Tys);
734 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
743 if (
Name.consume_front(
"mve.")) {
745 if (
Name ==
"vctp64") {
746 if (cast<FixedVectorType>(
F->getReturnType())->getNumElements() == 4) {
756 if (
Name.consume_back(
".v4i1")) {
758 if (
Name.consume_back(
".predicated.v2i64.v4i32"))
760 return Name ==
"mull.int" ||
Name ==
"vqdmull";
762 if (
Name.consume_back(
".v2i64")) {
764 bool IsGather =
Name.consume_front(
"vldr.gather.");
765 if (IsGather ||
Name.consume_front(
"vstr.scatter.")) {
766 if (
Name.consume_front(
"base.")) {
768 Name.consume_front(
"wb.");
771 return Name ==
"predicated.v2i64";
774 if (
Name.consume_front(
"offset.predicated."))
775 return Name == (IsGather ?
"v2i64.p0i64" :
"p0i64.v2i64") ||
776 Name == (IsGather ?
"v2i64.p0" :
"p0.v2i64");
789 if (
Name.consume_front(
"cde.vcx")) {
791 if (
Name.consume_back(
".predicated.v2i64.v4i1"))
808 F->arg_begin()->getType());
812 if (
Name.starts_with(
"addp")) {
814 if (
F->arg_size() != 2)
816 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
817 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
819 Intrinsic::aarch64_neon_faddp, Ty);
825 if (
Name.consume_front(
"sve.")) {
827 if (
Name.consume_front(
"bf")) {
828 if (
Name.consume_back(
".lane")) {
832 .
Case(
"dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
833 .
Case(
"mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
834 .
Case(
"mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
845 if (
Name.consume_front(
"ld")) {
847 static const Regex LdRegex(
"^[234](.nxv[a-z0-9]+|$)");
850 dyn_cast<VectorType>(
F->getReturnType())->getElementType();
851 ElementCount EC = dyn_cast<VectorType>(
F->arg_begin()->getType())
853 Type *Ty = VectorType::get(ScalarTy, EC);
855 Intrinsic::aarch64_sve_ld2_sret,
856 Intrinsic::aarch64_sve_ld3_sret,
857 Intrinsic::aarch64_sve_ld4_sret,
860 LoadIDs[
Name[0] -
'2'], Ty);
866 if (
Name.consume_front(
"tuple.")) {
868 if (
Name.starts_with(
"get")) {
870 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
872 Intrinsic::vector_extract, Tys);
876 if (
Name.starts_with(
"set")) {
878 auto Args =
F->getFunctionType()->params();
879 Type *Tys[] = {Args[0], Args[2], Args[1]};
881 Intrinsic::vector_insert, Tys);
885 static const Regex CreateTupleRegex(
"^create[234](.nxv[a-z0-9]+|$)");
888 auto Args =
F->getFunctionType()->params();
889 Type *Tys[] = {
F->getReturnType(), Args[1]};
891 Intrinsic::vector_insert, Tys);
903 if (
Name.consume_front(
"abs."))
905 .
Case(
"bf16", Intrinsic::nvvm_abs_bf16)
906 .
Case(
"bf16x2", Intrinsic::nvvm_abs_bf16x2)
909 if (
Name.consume_front(
"fma.rn."))
911 .
Case(
"bf16", Intrinsic::nvvm_fma_rn_bf16)
912 .
Case(
"bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
913 .
Case(
"ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)
914 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)
915 .
Case(
"ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)
916 .
Case(
"ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)
917 .
Case(
"ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)
918 .
Case(
"ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)
919 .
Case(
"relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
920 .
Case(
"relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
921 .
Case(
"sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)
922 .
Case(
"sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)
925 if (
Name.consume_front(
"fmax."))
927 .
Case(
"bf16", Intrinsic::nvvm_fmax_bf16)
928 .
Case(
"bf16x2", Intrinsic::nvvm_fmax_bf16x2)
929 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
930 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
931 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
932 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
933 .
Case(
"ftz.nan.xorsign.abs.bf16",
934 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
935 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
936 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
937 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
938 .
Case(
"ftz.xorsign.abs.bf16x2",
939 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
940 .
Case(
"nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
941 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
942 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
943 .
Case(
"nan.xorsign.abs.bf16x2",
944 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
945 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
946 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
949 if (
Name.consume_front(
"fmin."))
951 .
Case(
"bf16", Intrinsic::nvvm_fmin_bf16)
952 .
Case(
"bf16x2", Intrinsic::nvvm_fmin_bf16x2)
953 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
954 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
955 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
956 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
957 .
Case(
"ftz.nan.xorsign.abs.bf16",
958 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
959 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
960 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
961 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
962 .
Case(
"ftz.xorsign.abs.bf16x2",
963 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
964 .
Case(
"nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
965 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
966 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
967 .
Case(
"nan.xorsign.abs.bf16x2",
968 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
969 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
970 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
973 if (
Name.consume_front(
"neg."))
975 .
Case(
"bf16", Intrinsic::nvvm_neg_bf16)
976 .
Case(
"bf16x2", Intrinsic::nvvm_neg_bf16x2)
983 assert(
F &&
"Illegal to upgrade a non-existent Function.");
988 if (!
Name.consume_front(
"llvm.") ||
Name.empty())
994 bool IsArm =
Name.consume_front(
"arm.");
995 if (IsArm ||
Name.consume_front(
"aarch64.")) {
1001 if (
Name.consume_front(
"amdgcn.")) {
1002 if (
Name ==
"alignbit") {
1005 {F->getReturnType()});
1009 if (
Name.consume_front(
"atomic.")) {
1010 if (
Name.starts_with(
"inc") ||
Name.starts_with(
"dec")) {
1019 if (
Name.starts_with(
"ldexp.")) {
1022 F->getParent(), Intrinsic::ldexp,
1023 {F->getReturnType(), F->getArg(1)->getType()});
1032 if (
F->arg_size() == 1) {
1040 F->arg_begin()->getType());
1045 if (
F->arg_size() == 2 &&
Name.equals(
"coro.end")) {
1054 if (
Name.consume_front(
"dbg.")) {
1055 if (
Name ==
"addr" || (
Name ==
"value" &&
F->arg_size() == 4)) {
1064 if (
Name.consume_front(
"experimental.vector.")) {
1066 .
StartsWith(
"extract.", Intrinsic::vector_extract)
1067 .
StartsWith(
"insert.", Intrinsic::vector_insert)
1070 const auto *FT =
F->getFunctionType();
1072 if (
ID == Intrinsic::vector_extract)
1076 if (
ID == Intrinsic::vector_insert)
1084 if (
Name.consume_front(
"reduce.")) {
1086 static const Regex R(
"^([a-z]+)\\.[a-z][0-9]+");
1089 .
Case(
"add", Intrinsic::vector_reduce_add)
1090 .
Case(
"mul", Intrinsic::vector_reduce_mul)
1091 .
Case(
"and", Intrinsic::vector_reduce_and)
1092 .
Case(
"or", Intrinsic::vector_reduce_or)
1093 .
Case(
"xor", Intrinsic::vector_reduce_xor)
1094 .
Case(
"smax", Intrinsic::vector_reduce_smax)
1095 .
Case(
"smin", Intrinsic::vector_reduce_smin)
1096 .
Case(
"umax", Intrinsic::vector_reduce_umax)
1097 .
Case(
"umin", Intrinsic::vector_reduce_umin)
1098 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
1099 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
1104 static const Regex R2(
"^v2\\.([a-z]+)\\.[fi][0-9]+");
1109 .
Case(
"fadd", Intrinsic::vector_reduce_fadd)
1110 .
Case(
"fmul", Intrinsic::vector_reduce_fmul)
1115 auto Args =
F->getFunctionType()->params();
1126 if (
Name.starts_with(
"flt.rounds")) {
1133 if (
Name.starts_with(
"invariant.group.barrier")) {
1135 auto Args =
F->getFunctionType()->params();
1136 Type* ObjectPtr[1] = {Args[0]};
1139 Intrinsic::launder_invariant_group, ObjectPtr);
1148 .StartsWith(
"memcpy.", Intrinsic::memcpy)
1149 .StartsWith(
"memmove.", Intrinsic::memmove)
1151 if (
F->arg_size() == 5) {
1155 F->getFunctionType()->params().slice(0, 3);
1160 if (
Name.starts_with(
"memset.") &&
F->arg_size() == 5) {
1163 const auto *FT =
F->getFunctionType();
1164 Type *ParamTypes[2] = {
1165 FT->getParamType(0),
1175 if (
Name.consume_front(
"nvvm.")) {
1177 if (
F->arg_size() == 1) {
1180 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
1181 .
Case(
"clz.i", Intrinsic::ctlz)
1182 .
Case(
"popc.i", Intrinsic::ctpop)
1186 {F->getReturnType()});
1192 if (!
F->getReturnType()->getScalarType()->isBFloatTy()) {
1204 bool Expand =
false;
1205 if (
Name.consume_front(
"abs."))
1207 Expand =
Name ==
"i" ||
Name ==
"ll";
1208 else if (
Name ==
"clz.ll" ||
Name ==
"popc.ll" ||
Name ==
"h2f")
1210 else if (
Name.consume_front(
"max.") ||
Name.consume_front(
"min."))
1214 else if (
Name.consume_front(
"atomic.load.add."))
1216 Expand =
Name.starts_with(
"f32.p") ||
Name.starts_with(
"f64.p");
1231 if (
Name.starts_with(
"objectsize.")) {
1232 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
1233 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
1245 if (
Name.starts_with(
"ptr.annotation.") &&
F->arg_size() == 4) {
1248 F->getParent(), Intrinsic::ptr_annotation,
1249 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1255 if (
Name.consume_front(
"riscv.")) {
1258 .
Case(
"aes32dsi", Intrinsic::riscv_aes32dsi)
1259 .
Case(
"aes32dsmi", Intrinsic::riscv_aes32dsmi)
1260 .
Case(
"aes32esi", Intrinsic::riscv_aes32esi)
1261 .
Case(
"aes32esmi", Intrinsic::riscv_aes32esmi)
1264 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1277 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1278 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1287 .
StartsWith(
"sha256sig0", Intrinsic::riscv_sha256sig0)
1288 .
StartsWith(
"sha256sig1", Intrinsic::riscv_sha256sig1)
1289 .
StartsWith(
"sha256sum0", Intrinsic::riscv_sha256sum0)
1290 .
StartsWith(
"sha256sum1", Intrinsic::riscv_sha256sum1)
1295 if (
F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1307 if (
Name ==
"stackprotectorcheck") {
1314 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
1317 F->getParent(), Intrinsic::var_annotation,
1318 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1325 if (
Name.consume_front(
"wasm.")) {
1328 .
StartsWith(
"fma.", Intrinsic::wasm_relaxed_madd)
1329 .
StartsWith(
"fms.", Intrinsic::wasm_relaxed_nmadd)
1330 .
StartsWith(
"laneselect.", Intrinsic::wasm_relaxed_laneselect)
1339 if (
Name.consume_front(
"dot.i8x16.i7x16.")) {
1341 .
Case(
"signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1343 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1361 auto *
ST = dyn_cast<StructType>(
F->getReturnType());
1362 if (ST && (!
ST->isLiteral() ||
ST->isPacked()) &&
1371 auto *FT =
F->getFunctionType();
1373 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1374 std::string
Name =
F->getName().str();
1377 Name,
F->getParent());
1388 if (Result != std::nullopt) {
1403 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1415 GV->
getName() ==
"llvm.global_dtors")) ||
1430 unsigned N =
Init->getNumOperands();
1431 std::vector<Constant *> NewCtors(
N);
1432 for (
unsigned i = 0; i !=
N; ++i) {
1433 auto Ctor = cast<Constant>(
Init->getOperand(i));
1435 Ctor->getAggregateElement(1),
1448 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1449 unsigned NumElts = ResultTy->getNumElements() * 8;
1463 for (
unsigned l = 0; l != NumElts; l += 16)
1464 for (
unsigned i = 0; i != 16; ++i) {
1465 unsigned Idx = NumElts + i - Shift;
1467 Idx -= NumElts - 16;
1468 Idxs[l + i] =
Idx + l;
1482 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1483 unsigned NumElts = ResultTy->getNumElements() * 8;
1497 for (
unsigned l = 0; l != NumElts; l += 16)
1498 for (
unsigned i = 0; i != 16; ++i) {
1499 unsigned Idx = i + Shift;
1501 Idx += NumElts - 16;
1502 Idxs[l + i] =
Idx + l;
1523 for (
unsigned i = 0; i != NumElts; ++i)
1535 if (
const auto *
C = dyn_cast<Constant>(Mask))
1536 if (
C->isAllOnesValue())
1540 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1547 if (
const auto *
C = dyn_cast<Constant>(Mask))
1548 if (
C->isAllOnesValue())
1552 Mask->getType()->getIntegerBitWidth());
1565 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1567 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1568 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1569 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1574 ShiftVal &= (NumElts - 1);
1583 if (ShiftVal > 16) {
1591 for (
unsigned l = 0; l < NumElts; l += 16) {
1592 for (
unsigned i = 0; i != 16; ++i) {
1593 unsigned Idx = ShiftVal + i;
1594 if (!IsVALIGN &&
Idx >= 16)
1595 Idx += NumElts - 16;
1596 Indices[l + i] =
Idx + l;
1601 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
1607 bool ZeroMask,
bool IndexForm) {
1613 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1614 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1615 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1616 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1617 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1618 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1619 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1620 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1621 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1622 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1623 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1624 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1625 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1626 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1627 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1628 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1629 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1630 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1631 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1632 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1633 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1634 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1635 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1636 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1637 else if (VecWidth == 128 && EltWidth == 16)
1638 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1639 else if (VecWidth == 256 && EltWidth == 16)
1640 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1641 else if (VecWidth == 512 && EltWidth == 16)
1642 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1643 else if (VecWidth == 128 && EltWidth == 8)
1644 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1645 else if (VecWidth == 256 && EltWidth == 8)
1646 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1647 else if (VecWidth == 512 && EltWidth == 8)
1648 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1684 bool IsRotateRight) {
1693 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1698 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1719 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1722 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1725 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1728 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1731 Pred = ICmpInst::ICMP_EQ;
1734 Pred = ICmpInst::ICMP_NE;
1750 bool IsShiftRight,
bool ZeroMask) {
1763 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1768 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1788 const Align Alignment =
1790 ?
Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1794 if (
const auto *
C = dyn_cast<Constant>(Mask))
1795 if (
C->isAllOnesValue())
1799 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1809 const Align Alignment =
1817 if (
const auto *
C = dyn_cast<Constant>(Mask))
1818 if (
C->isAllOnesValue())
1822 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1846 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1853 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1869 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1871 const auto *
C = dyn_cast<Constant>(Mask);
1872 if (!
C || !
C->isAllOnesValue())
1878 for (
unsigned i = 0; i != NumElts; ++i)
1880 for (
unsigned i = NumElts; i != 8; ++i)
1881 Indices[i] = NumElts + i % NumElts;
1892 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1898 }
else if (
CC == 7) {
1905 case 0: Pred = ICmpInst::ICMP_EQ;
break;
1906 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
1907 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
1908 case 4: Pred = ICmpInst::ICMP_NE;
break;
1909 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
1910 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
1946 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
1948 return Builder.
CreateSExt(Mask, ReturnOp,
"vpmovm2");
1959 if (
Name.starts_with(
"max.p")) {
1960 if (VecWidth == 128 && EltWidth == 32)
1961 IID = Intrinsic::x86_sse_max_ps;
1962 else if (VecWidth == 128 && EltWidth == 64)
1963 IID = Intrinsic::x86_sse2_max_pd;
1964 else if (VecWidth == 256 && EltWidth == 32)
1965 IID = Intrinsic::x86_avx_max_ps_256;
1966 else if (VecWidth == 256 && EltWidth == 64)
1967 IID = Intrinsic::x86_avx_max_pd_256;
1970 }
else if (
Name.starts_with(
"min.p")) {
1971 if (VecWidth == 128 && EltWidth == 32)
1972 IID = Intrinsic::x86_sse_min_ps;
1973 else if (VecWidth == 128 && EltWidth == 64)
1974 IID = Intrinsic::x86_sse2_min_pd;
1975 else if (VecWidth == 256 && EltWidth == 32)
1976 IID = Intrinsic::x86_avx_min_ps_256;
1977 else if (VecWidth == 256 && EltWidth == 64)
1978 IID = Intrinsic::x86_avx_min_pd_256;
1981 }
else if (
Name.starts_with(
"pshuf.b.")) {
1982 if (VecWidth == 128)
1983 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1984 else if (VecWidth == 256)
1985 IID = Intrinsic::x86_avx2_pshuf_b;
1986 else if (VecWidth == 512)
1987 IID = Intrinsic::x86_avx512_pshuf_b_512;
1990 }
else if (
Name.starts_with(
"pmul.hr.sw.")) {
1991 if (VecWidth == 128)
1992 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1993 else if (VecWidth == 256)
1994 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1995 else if (VecWidth == 512)
1996 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1999 }
else if (
Name.starts_with(
"pmulh.w.")) {
2000 if (VecWidth == 128)
2001 IID = Intrinsic::x86_sse2_pmulh_w;
2002 else if (VecWidth == 256)
2003 IID = Intrinsic::x86_avx2_pmulh_w;
2004 else if (VecWidth == 512)
2005 IID = Intrinsic::x86_avx512_pmulh_w_512;
2008 }
else if (
Name.starts_with(
"pmulhu.w.")) {
2009 if (VecWidth == 128)
2010 IID = Intrinsic::x86_sse2_pmulhu_w;
2011 else if (VecWidth == 256)
2012 IID = Intrinsic::x86_avx2_pmulhu_w;
2013 else if (VecWidth == 512)
2014 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2017 }
else if (
Name.starts_with(
"pmaddw.d.")) {
2018 if (VecWidth == 128)
2019 IID = Intrinsic::x86_sse2_pmadd_wd;
2020 else if (VecWidth == 256)
2021 IID = Intrinsic::x86_avx2_pmadd_wd;
2022 else if (VecWidth == 512)
2023 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2026 }
else if (
Name.starts_with(
"pmaddubs.w.")) {
2027 if (VecWidth == 128)
2028 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2029 else if (VecWidth == 256)
2030 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2031 else if (VecWidth == 512)
2032 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2035 }
else if (
Name.starts_with(
"packsswb.")) {
2036 if (VecWidth == 128)
2037 IID = Intrinsic::x86_sse2_packsswb_128;
2038 else if (VecWidth == 256)
2039 IID = Intrinsic::x86_avx2_packsswb;
2040 else if (VecWidth == 512)
2041 IID = Intrinsic::x86_avx512_packsswb_512;
2044 }
else if (
Name.starts_with(
"packssdw.")) {
2045 if (VecWidth == 128)
2046 IID = Intrinsic::x86_sse2_packssdw_128;
2047 else if (VecWidth == 256)
2048 IID = Intrinsic::x86_avx2_packssdw;
2049 else if (VecWidth == 512)
2050 IID = Intrinsic::x86_avx512_packssdw_512;
2053 }
else if (
Name.starts_with(
"packuswb.")) {
2054 if (VecWidth == 128)
2055 IID = Intrinsic::x86_sse2_packuswb_128;
2056 else if (VecWidth == 256)
2057 IID = Intrinsic::x86_avx2_packuswb;
2058 else if (VecWidth == 512)
2059 IID = Intrinsic::x86_avx512_packuswb_512;
2062 }
else if (
Name.starts_with(
"packusdw.")) {
2063 if (VecWidth == 128)
2064 IID = Intrinsic::x86_sse41_packusdw;
2065 else if (VecWidth == 256)
2066 IID = Intrinsic::x86_avx2_packusdw;
2067 else if (VecWidth == 512)
2068 IID = Intrinsic::x86_avx512_packusdw_512;
2071 }
else if (
Name.starts_with(
"vpermilvar.")) {
2072 if (VecWidth == 128 && EltWidth == 32)
2073 IID = Intrinsic::x86_avx_vpermilvar_ps;
2074 else if (VecWidth == 128 && EltWidth == 64)
2075 IID = Intrinsic::x86_avx_vpermilvar_pd;
2076 else if (VecWidth == 256 && EltWidth == 32)
2077 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2078 else if (VecWidth == 256 && EltWidth == 64)
2079 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2080 else if (VecWidth == 512 && EltWidth == 32)
2081 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2082 else if (VecWidth == 512 && EltWidth == 64)
2083 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2086 }
else if (
Name ==
"cvtpd2dq.256") {
2087 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2088 }
else if (
Name ==
"cvtpd2ps.256") {
2089 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2090 }
else if (
Name ==
"cvttpd2dq.256") {
2091 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2092 }
else if (
Name ==
"cvttps2dq.128") {
2093 IID = Intrinsic::x86_sse2_cvttps2dq;
2094 }
else if (
Name ==
"cvttps2dq.256") {
2095 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2096 }
else if (
Name.starts_with(
"permvar.")) {
2098 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2099 IID = Intrinsic::x86_avx2_permps;
2100 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2101 IID = Intrinsic::x86_avx2_permd;
2102 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2103 IID = Intrinsic::x86_avx512_permvar_df_256;
2104 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2105 IID = Intrinsic::x86_avx512_permvar_di_256;
2106 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2107 IID = Intrinsic::x86_avx512_permvar_sf_512;
2108 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2109 IID = Intrinsic::x86_avx512_permvar_si_512;
2110 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2111 IID = Intrinsic::x86_avx512_permvar_df_512;
2112 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2113 IID = Intrinsic::x86_avx512_permvar_di_512;
2114 else if (VecWidth == 128 && EltWidth == 16)
2115 IID = Intrinsic::x86_avx512_permvar_hi_128;
2116 else if (VecWidth == 256 && EltWidth == 16)
2117 IID = Intrinsic::x86_avx512_permvar_hi_256;
2118 else if (VecWidth == 512 && EltWidth == 16)
2119 IID = Intrinsic::x86_avx512_permvar_hi_512;
2120 else if (VecWidth == 128 && EltWidth == 8)
2121 IID = Intrinsic::x86_avx512_permvar_qi_128;
2122 else if (VecWidth == 256 && EltWidth == 8)
2123 IID = Intrinsic::x86_avx512_permvar_qi_256;
2124 else if (VecWidth == 512 && EltWidth == 8)
2125 IID = Intrinsic::x86_avx512_permvar_qi_512;
2128 }
else if (
Name.starts_with(
"dbpsadbw.")) {
2129 if (VecWidth == 128)
2130 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2131 else if (VecWidth == 256)
2132 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2133 else if (VecWidth == 512)
2134 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2137 }
else if (
Name.starts_with(
"pmultishift.qb.")) {
2138 if (VecWidth == 128)
2139 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2140 else if (VecWidth == 256)
2141 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2142 else if (VecWidth == 512)
2143 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2146 }
else if (
Name.starts_with(
"conflict.")) {
2147 if (
Name[9] ==
'd' && VecWidth == 128)
2148 IID = Intrinsic::x86_avx512_conflict_d_128;
2149 else if (
Name[9] ==
'd' && VecWidth == 256)
2150 IID = Intrinsic::x86_avx512_conflict_d_256;
2151 else if (
Name[9] ==
'd' && VecWidth == 512)
2152 IID = Intrinsic::x86_avx512_conflict_d_512;
2153 else if (
Name[9] ==
'q' && VecWidth == 128)
2154 IID = Intrinsic::x86_avx512_conflict_q_128;
2155 else if (
Name[9] ==
'q' && VecWidth == 256)
2156 IID = Intrinsic::x86_avx512_conflict_q_256;
2157 else if (
Name[9] ==
'q' && VecWidth == 512)
2158 IID = Intrinsic::x86_avx512_conflict_q_512;
2161 }
else if (
Name.starts_with(
"pavg.")) {
2162 if (
Name[5] ==
'b' && VecWidth == 128)
2163 IID = Intrinsic::x86_sse2_pavg_b;
2164 else if (
Name[5] ==
'b' && VecWidth == 256)
2165 IID = Intrinsic::x86_avx2_pavg_b;
2166 else if (
Name[5] ==
'b' && VecWidth == 512)
2167 IID = Intrinsic::x86_avx512_pavg_b_512;
2168 else if (
Name[5] ==
'w' && VecWidth == 128)
2169 IID = Intrinsic::x86_sse2_pavg_w;
2170 else if (
Name[5] ==
'w' && VecWidth == 256)
2171 IID = Intrinsic::x86_avx2_pavg_w;
2172 else if (
Name[5] ==
'w' && VecWidth == 512)
2173 IID = Intrinsic::x86_avx512_pavg_w_512;
2194 if (AsmStr->find(
"mov\tfp") == 0 &&
2195 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
2196 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
2197 AsmStr->replace(Pos, 1,
";");
2203 if (
Name ==
"mve.vctp64.old") {
2211 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2212 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2216 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2217 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2219 }
else if (
Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2220 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2221 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2222 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2224 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2225 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2226 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2227 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2229 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2230 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2231 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
2232 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
2233 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
2234 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
2235 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
2236 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
2237 std::vector<Type *> Tys;
2241 case Intrinsic::arm_mve_mull_int_predicated:
2242 case Intrinsic::arm_mve_vqdmull_predicated:
2243 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2246 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2247 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2248 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2252 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2256 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2260 case Intrinsic::arm_cde_vcx1q_predicated:
2261 case Intrinsic::arm_cde_vcx1qa_predicated:
2262 case Intrinsic::arm_cde_vcx2q_predicated:
2263 case Intrinsic::arm_cde_vcx2qa_predicated:
2264 case Intrinsic::arm_cde_vcx3q_predicated:
2265 case Intrinsic::arm_cde_vcx3qa_predicated:
2272 std::vector<Value *> Ops;
2274 Type *Ty =
Op->getType();
2278 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2279 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2283 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2297 const bool IsInc =
Name.starts_with(
"atomic.inc.");
2298 if (IsInc ||
Name.starts_with(
"atomic.dec.")) {
2313 if (Order == AtomicOrdering::NotAtomic ||
2314 Order == AtomicOrdering::Unordered)
2315 Order = AtomicOrdering::SequentiallyConsistent;
2319 SyncScope::ID SSID =
F->getContext().getOrInsertSyncScopeID(
"agent");
2323 if (!VolatileArg || !VolatileArg->
isZero())
2349 assert(
Name.starts_with(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
2352 bool IsX86 =
Name.consume_front(
"x86.");
2353 bool IsNVVM =
Name.consume_front(
"nvvm.");
2354 bool IsARM =
Name.consume_front(
"arm.");
2355 bool IsAMDGCN =
Name.consume_front(
"amdgcn.");
2357 if (IsX86 &&
Name.starts_with(
"sse4a.movnt.")) {
2368 Type *SrcEltTy = cast<VectorType>(Arg1->
getType())->getElementType();
2369 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2375 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2382 if (IsX86 && (
Name.starts_with(
"avx.movnt.") ||
2383 Name.starts_with(
"avx512.storent."))) {
2394 PointerType::getUnqual(Arg1->
getType()),
2399 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2406 if (IsX86 &&
Name ==
"sse2.storel.dq") {
2414 PointerType::getUnqual(Elt->
getType()),
2423 if (IsX86 && (
Name.starts_with(
"sse.storeu.") ||
2424 Name.starts_with(
"sse2.storeu.") ||
2425 Name.starts_with(
"avx.storeu."))) {
2430 PointerType::getUnqual(Arg1->
getType()),
2439 if (IsX86 &&
Name ==
"avx512.mask.store.ss") {
2449 if (IsX86 &&
Name.starts_with(
"avx512.mask.store")) {
2462 if (IsX86 && (
Name.starts_with(
"sse2.pcmp") ||
2463 Name.starts_with(
"avx2.pcmp"))) {
2465 bool CmpEq =
Name[9] ==
'e';
2466 Rep = Builder.
CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2469 }
else if (IsX86 && (
Name.starts_with(
"avx512.broadcastm"))) {
2477 }
else if (IsX86 && (
Name ==
"sse.sqrt.ss" ||
2478 Name ==
"sse2.sqrt.sd")) {
2482 Intrinsic::sqrt, Elt0->
getType());
2485 }
else if (IsX86 && (
Name.starts_with(
"avx.sqrt.p") ||
2486 Name.starts_with(
"sse2.sqrt.p") ||
2487 Name.starts_with(
"sse.sqrt.p"))) {
2491 {CI->getArgOperand(0)});
2492 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.sqrt.p"))) {
2495 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2497 : Intrinsic::x86_avx512_sqrt_pd_512;
2506 {CI->getArgOperand(0)});
2510 }
else if (IsX86 && (
Name.starts_with(
"avx512.ptestm") ||
2511 Name.starts_with(
"avx512.ptestnm"))) {
2519 Name.starts_with(
"avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2522 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pbroadcast"))){
2528 }
else if (IsX86 && (
Name.starts_with(
"avx512.kunpck"))) {
2533 for (
unsigned i = 0; i != NumElts; ++i)
2546 }
else if (IsX86 &&
Name ==
"avx512.kand.w") {
2551 }
else if (IsX86 &&
Name ==
"avx512.kandn.w") {
2557 }
else if (IsX86 &&
Name ==
"avx512.kor.w") {
2562 }
else if (IsX86 &&
Name ==
"avx512.kxor.w") {
2567 }
else if (IsX86 &&
Name ==
"avx512.kxnor.w") {
2573 }
else if (IsX86 &&
Name ==
"avx512.knot.w") {
2578 (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w")) {
2584 if (
Name[14] ==
'c')
2585 C = ConstantInt::getAllOnesValue(Builder.
getInt16Ty());
2587 C = ConstantInt::getNullValue(Builder.
getInt16Ty());
2590 }
else if (IsX86 && (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2591 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2592 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2593 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd")) {
2596 ConstantInt::get(I32Ty, 0));
2598 ConstantInt::get(I32Ty, 0));
2600 if (
Name.contains(
".add."))
2602 else if (
Name.contains(
".sub."))
2604 else if (
Name.contains(
".mul."))
2609 ConstantInt::get(I32Ty, 0));
2610 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pcmp")) {
2612 bool CmpEq =
Name[16] ==
'e';
2614 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.vpshufbitqmb.")) {
2620 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
break;
2621 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
break;
2622 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
break;
2626 { CI->getOperand(0), CI->getArgOperand(1) });
2628 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.fpclass.p")) {
2633 if (VecWidth == 128 && EltWidth == 32)
2634 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2635 else if (VecWidth == 256 && EltWidth == 32)
2636 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2637 else if (VecWidth == 512 && EltWidth == 32)
2638 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2639 else if (VecWidth == 128 && EltWidth == 64)
2640 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2641 else if (VecWidth == 256 && EltWidth == 64)
2642 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2643 else if (VecWidth == 512 && EltWidth == 64)
2644 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2649 { CI->getOperand(0), CI->getArgOperand(1) });
2651 }
else if (IsX86 &&
Name.starts_with(
"avx512.cmp.p")) {
2653 Type *OpTy = Args[0]->getType();
2657 if (VecWidth == 128 && EltWidth == 32)
2658 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2659 else if (VecWidth == 256 && EltWidth == 32)
2660 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2661 else if (VecWidth == 512 && EltWidth == 32)
2662 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2663 else if (VecWidth == 128 && EltWidth == 64)
2664 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2665 else if (VecWidth == 256 && EltWidth == 64)
2666 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2667 else if (VecWidth == 512 && EltWidth == 64)
2668 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2673 if (VecWidth == 512)
2675 Args.push_back(Mask);
2679 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.cmp.")) {
2681 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2683 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.ucmp.")) {
2684 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2686 }
else if (IsX86 && (
Name.starts_with(
"avx512.cvtb2mask.") ||
2687 Name.starts_with(
"avx512.cvtw2mask.") ||
2688 Name.starts_with(
"avx512.cvtd2mask.") ||
2689 Name.starts_with(
"avx512.cvtq2mask."))) {
2692 Rep = Builder.
CreateICmp(ICmpInst::ICMP_SLT,
Op, Zero);
2694 }
else if(IsX86 && (
Name ==
"ssse3.pabs.b.128" ||
2695 Name ==
"ssse3.pabs.w.128" ||
2696 Name ==
"ssse3.pabs.d.128" ||
2697 Name.starts_with(
"avx2.pabs") ||
2698 Name.starts_with(
"avx512.mask.pabs"))) {
2700 }
else if (IsX86 && (
Name ==
"sse41.pmaxsb" ||
2701 Name ==
"sse2.pmaxs.w" ||
2702 Name ==
"sse41.pmaxsd" ||
2703 Name.starts_with(
"avx2.pmaxs") ||
2704 Name.starts_with(
"avx512.mask.pmaxs"))) {
2706 }
else if (IsX86 && (
Name ==
"sse2.pmaxu.b" ||
2707 Name ==
"sse41.pmaxuw" ||
2708 Name ==
"sse41.pmaxud" ||
2709 Name.starts_with(
"avx2.pmaxu") ||
2710 Name.starts_with(
"avx512.mask.pmaxu"))) {
2712 }
else if (IsX86 && (
Name ==
"sse41.pminsb" ||
2713 Name ==
"sse2.pmins.w" ||
2714 Name ==
"sse41.pminsd" ||
2715 Name.starts_with(
"avx2.pmins") ||
2716 Name.starts_with(
"avx512.mask.pmins"))) {
2718 }
else if (IsX86 && (
Name ==
"sse2.pminu.b" ||
2719 Name ==
"sse41.pminuw" ||
2720 Name ==
"sse41.pminud" ||
2721 Name.starts_with(
"avx2.pminu") ||
2722 Name.starts_with(
"avx512.mask.pminu"))) {
2724 }
else if (IsX86 && (
Name ==
"sse2.pmulu.dq" ||
2725 Name ==
"avx2.pmulu.dq" ||
2726 Name ==
"avx512.pmulu.dq.512" ||
2727 Name.starts_with(
"avx512.mask.pmulu.dq."))) {
2729 }
else if (IsX86 && (
Name ==
"sse41.pmuldq" ||
2730 Name ==
"avx2.pmul.dq" ||
2731 Name ==
"avx512.pmul.dq.512" ||
2732 Name.starts_with(
"avx512.mask.pmul.dq."))) {
2734 }
else if (IsX86 && (
Name ==
"sse.cvtsi2ss" ||
2735 Name ==
"sse2.cvtsi2sd" ||
2736 Name ==
"sse.cvtsi642ss" ||
2737 Name ==
"sse2.cvtsi642sd")) {
2740 cast<VectorType>(CI->
getType())->getElementType());
2742 }
else if (IsX86 &&
Name ==
"avx512.cvtusi2sd") {
2745 cast<VectorType>(CI->
getType())->getElementType());
2747 }
else if (IsX86 &&
Name ==
"sse2.cvtss2sd") {
2750 Rep, cast<VectorType>(CI->
getType())->getElementType());
2752 }
else if (IsX86 && (
Name ==
"sse2.cvtdq2pd" ||
2753 Name ==
"sse2.cvtdq2ps" ||
2754 Name ==
"avx.cvtdq2.pd.256" ||
2755 Name ==
"avx.cvtdq2.ps.256" ||
2756 Name.starts_with(
"avx512.mask.cvtdq2pd.") ||
2757 Name.starts_with(
"avx512.mask.cvtudq2pd.") ||
2758 Name.starts_with(
"avx512.mask.cvtdq2ps.") ||
2759 Name.starts_with(
"avx512.mask.cvtudq2ps.") ||
2760 Name.starts_with(
"avx512.mask.cvtqq2pd.") ||
2761 Name.starts_with(
"avx512.mask.cvtuqq2pd.") ||
2762 Name ==
"avx512.mask.cvtqq2ps.256" ||
2763 Name ==
"avx512.mask.cvtqq2ps.512" ||
2764 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2765 Name ==
"avx512.mask.cvtuqq2ps.512" ||
2766 Name ==
"sse2.cvtps2pd" ||
2767 Name ==
"avx.cvt.ps2.pd.256" ||
2768 Name ==
"avx512.mask.cvtps2pd.128" ||
2769 Name ==
"avx512.mask.cvtps2pd.256")) {
2770 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2772 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2774 unsigned NumDstElts = DstTy->getNumElements();
2775 if (NumDstElts < SrcTy->getNumElements()) {
2776 assert(NumDstElts == 2 &&
"Unexpected vector size");
2780 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2781 bool IsUnsigned =
Name.contains(
"cvtu");
2783 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtps2pd");
2786 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2787 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2788 : Intrinsic::x86_avx512_sitofp_round;
2793 Rep = IsUnsigned ? Builder.
CreateUIToFP(Rep, DstTy,
"cvt")
2800 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vcvtph2ps.") ||
2801 Name.starts_with(
"vcvtph2ps."))) {
2802 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2804 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2805 unsigned NumDstElts = DstTy->getNumElements();
2806 if (NumDstElts != SrcTy->getNumElements()) {
2807 assert(NumDstElts == 4 &&
"Unexpected vector size");
2812 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtph2ps");
2816 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.load")) {
2822 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.expand.load.")) {
2823 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2824 Type *PtrTy = ResultTy->getElementType();
2831 ResultTy->getNumElements());
2834 Intrinsic::masked_expandload,
2837 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.compress.store.")) {
2839 Type *PtrTy = ResultTy->getElementType();
2847 cast<FixedVectorType>(ResultTy)->getNumElements());
2850 Intrinsic::masked_compressstore,
2853 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.compress.") ||
2854 Name.starts_with(
"avx512.mask.expand."))) {
2855 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2858 ResultTy->getNumElements());
2860 bool IsCompress =
Name[12] ==
'c';
2861 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2862 : Intrinsic::x86_avx512_mask_expand;
2866 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcom")) {
2868 if (
Name.ends_with(
"ub") ||
Name.ends_with(
"uw") ||
Name.ends_with(
"ud") ||
2869 Name.ends_with(
"uq"))
2871 else if (
Name.ends_with(
"b") ||
Name.ends_with(
"w") ||
Name.ends_with(
"d") ||
2872 Name.ends_with(
"q"))
2879 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2882 if (
Name.starts_with(
"lt"))
2884 else if (
Name.starts_with(
"le"))
2886 else if (
Name.starts_with(
"gt"))
2888 else if (
Name.starts_with(
"ge"))
2890 else if (
Name.starts_with(
"eq"))
2892 else if (
Name.starts_with(
"ne"))
2894 else if (
Name.starts_with(
"false"))
2896 else if (
Name.starts_with(
"true"))
2903 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcmov")) {
2908 Rep = Builder.
CreateOr(Sel0, Sel1);
2909 }
else if (IsX86 && (
Name.starts_with(
"xop.vprot") ||
2910 Name.starts_with(
"avx512.prol") ||
2911 Name.starts_with(
"avx512.mask.prol"))) {
2913 }
else if (IsX86 && (
Name.starts_with(
"avx512.pror") ||
2914 Name.starts_with(
"avx512.mask.pror"))) {
2916 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshld.") ||
2917 Name.starts_with(
"avx512.mask.vpshld") ||
2918 Name.starts_with(
"avx512.maskz.vpshld"))) {
2919 bool ZeroMask =
Name[11] ==
'z';
2921 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshrd.") ||
2922 Name.starts_with(
"avx512.mask.vpshrd") ||
2923 Name.starts_with(
"avx512.maskz.vpshrd"))) {
2924 bool ZeroMask =
Name[11] ==
'z';
2926 }
else if (IsX86 &&
Name ==
"sse42.crc32.64.8") {
2928 Intrinsic::x86_sse42_crc32_32_8);
2932 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcast.s") ||
2933 Name.starts_with(
"avx512.vbroadcast.s"))) {
2935 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2936 Type *EltTy = VecTy->getElementType();
2937 unsigned EltNum = VecTy->getNumElements();
2941 for (
unsigned I = 0;
I < EltNum; ++
I)
2943 ConstantInt::get(I32Ty,
I));
2944 }
else if (IsX86 && (
Name.starts_with(
"sse41.pmovsx") ||
2945 Name.starts_with(
"sse41.pmovzx") ||
2946 Name.starts_with(
"avx2.pmovsx") ||
2947 Name.starts_with(
"avx2.pmovzx") ||
2948 Name.starts_with(
"avx512.mask.pmovsx") ||
2949 Name.starts_with(
"avx512.mask.pmovzx"))) {
2950 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2951 unsigned NumDstElts = DstTy->getNumElements();
2955 for (
unsigned i = 0; i != NumDstElts; ++i)
2961 bool DoSext =
Name.contains(
"pmovsx");
2968 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
2969 Name ==
"avx512.mask.pmov.qd.512" ||
2970 Name ==
"avx512.mask.pmov.wb.256" ||
2971 Name ==
"avx512.mask.pmov.wb.512") {
2976 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcastf128") ||
2977 Name ==
"avx2.vbroadcasti128")) {
2979 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
2983 PointerType::getUnqual(VT));
2985 if (NumSrcElts == 2)
2990 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.shuf.i") ||
2991 Name.starts_with(
"avx512.mask.shuf.f"))) {
2992 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2996 unsigned ControlBitsMask = NumLanes - 1;
2997 unsigned NumControlBits = NumLanes / 2;
3000 for (
unsigned l = 0; l != NumLanes; ++l) {
3001 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3003 if (l >= NumLanes / 2)
3004 LaneMask += NumLanes;
3005 for (
unsigned i = 0; i != NumElementsInLane; ++i)
3006 ShuffleMask.
push_back(LaneMask * NumElementsInLane + i);
3012 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.broadcastf") ||
3013 Name.starts_with(
"avx512.mask.broadcasti"))) {
3014 unsigned NumSrcElts =
3017 unsigned NumDstElts =
3018 cast<FixedVectorType>(CI->
getType())->getNumElements();
3021 for (
unsigned i = 0; i != NumDstElts; ++i)
3022 ShuffleMask[i] = i % NumSrcElts;
3029 }
else if (IsX86 && (
Name.starts_with(
"avx2.pbroadcast") ||
3030 Name.starts_with(
"avx2.vbroadcast") ||
3031 Name.starts_with(
"avx512.pbroadcast") ||
3032 Name.starts_with(
"avx512.mask.broadcast.s"))) {
3044 }
else if (IsX86 && (
Name.starts_with(
"sse2.padds.") ||
3045 Name.starts_with(
"avx2.padds.") ||
3046 Name.starts_with(
"avx512.padds.") ||
3047 Name.starts_with(
"avx512.mask.padds."))) {
3049 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubs.") ||
3050 Name.starts_with(
"avx2.psubs.") ||
3051 Name.starts_with(
"avx512.psubs.") ||
3052 Name.starts_with(
"avx512.mask.psubs."))) {
3054 }
else if (IsX86 && (
Name.starts_with(
"sse2.paddus.") ||
3055 Name.starts_with(
"avx2.paddus.") ||
3056 Name.starts_with(
"avx512.mask.paddus."))) {
3058 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubus.") ||
3059 Name.starts_with(
"avx2.psubus.") ||
3060 Name.starts_with(
"avx512.mask.psubus."))) {
3062 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.palignr.")) {
3067 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.valign.")) {
3072 }
else if (IsX86 && (
Name ==
"sse2.psll.dq" ||
3073 Name ==
"avx2.psll.dq")) {
3075 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3078 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq" ||
3079 Name ==
"avx2.psrl.dq")) {
3081 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3084 }
else if (IsX86 && (
Name ==
"sse2.psll.dq.bs" ||
3085 Name ==
"avx2.psll.dq.bs" ||
3086 Name ==
"avx512.psll.dq.512")) {
3088 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3090 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq.bs" ||
3091 Name ==
"avx2.psrl.dq.bs" ||
3092 Name ==
"avx512.psrl.dq.512")) {
3094 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3096 }
else if (IsX86 && (
Name ==
"sse41.pblendw" ||
3097 Name.starts_with(
"sse41.blendp") ||
3098 Name.starts_with(
"avx.blend.p") ||
3099 Name ==
"avx2.pblendw" ||
3100 Name.starts_with(
"avx2.pblendd."))) {
3103 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3104 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3105 unsigned NumElts = VecTy->getNumElements();
3108 for (
unsigned i = 0; i != NumElts; ++i)
3109 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
3112 }
else if (IsX86 && (
Name.starts_with(
"avx.vinsertf128.") ||
3113 Name ==
"avx2.vinserti128" ||
3114 Name.starts_with(
"avx512.mask.insert"))) {
3117 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3118 unsigned DstNumElts =
3119 cast<FixedVectorType>(CI->
getType())->getNumElements();
3120 unsigned SrcNumElts =
3121 cast<FixedVectorType>(Op1->
getType())->getNumElements();
3122 unsigned Scale = DstNumElts / SrcNumElts;
3129 for (
unsigned i = 0; i != SrcNumElts; ++i)
3131 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
3132 Idxs[i] = SrcNumElts;
3147 for (
unsigned i = 0; i != DstNumElts; ++i)
3150 for (
unsigned i = 0; i != SrcNumElts; ++i)
3151 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3158 }
else if (IsX86 && (
Name.starts_with(
"avx.vextractf128.") ||
3159 Name ==
"avx2.vextracti128" ||
3160 Name.starts_with(
"avx512.mask.vextract"))) {
3162 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3163 unsigned DstNumElts =
3164 cast<FixedVectorType>(CI->
getType())->getNumElements();
3165 unsigned SrcNumElts =
3166 cast<FixedVectorType>(Op0->
getType())->getNumElements();
3167 unsigned Scale = SrcNumElts / DstNumElts;
3174 for (
unsigned i = 0; i != DstNumElts; ++i) {
3175 Idxs[i] = i + (Imm * DstNumElts);
3183 }
else if (!IsX86 &&
Name ==
"stackprotectorcheck") {
3185 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.perm.df.") ||
3186 Name.starts_with(
"avx512.mask.perm.di."))) {
3188 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3189 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3190 unsigned NumElts = VecTy->getNumElements();
3193 for (
unsigned i = 0; i != NumElts; ++i)
3194 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3201 }
else if (IsX86 && (
Name.starts_with(
"avx.vperm2f128.") ||
3202 Name ==
"avx2.vperm2i128")) {
3211 uint8_t Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3213 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3214 unsigned HalfSize = NumElts / 2;
3226 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3227 for (
unsigned i = 0; i < HalfSize; ++i)
3228 ShuffleMask[i] = StartIndex + i;
3231 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3232 for (
unsigned i = 0; i < HalfSize; ++i)
3233 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3237 }
else if (IsX86 && (
Name.starts_with(
"avx.vpermil.") ||
3238 Name ==
"sse2.pshuf.d" ||
3239 Name.starts_with(
"avx512.mask.vpermil.p") ||
3240 Name.starts_with(
"avx512.mask.pshuf.d."))) {
3242 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3243 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3244 unsigned NumElts = VecTy->getNumElements();
3246 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3247 unsigned IdxMask = ((1 << IdxSize) - 1);
3253 for (
unsigned i = 0; i != NumElts; ++i)
3254 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3261 }
else if (IsX86 && (
Name ==
"sse2.pshufl.w" ||
3262 Name.starts_with(
"avx512.mask.pshufl.w."))) {
3264 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3265 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3268 for (
unsigned l = 0; l != NumElts; l += 8) {
3269 for (
unsigned i = 0; i != 4; ++i)
3270 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3271 for (
unsigned i = 4; i != 8; ++i)
3272 Idxs[i + l] = i + l;
3280 }
else if (IsX86 && (
Name ==
"sse2.pshufh.w" ||
3281 Name.starts_with(
"avx512.mask.pshufh.w."))) {
3283 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3284 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3287 for (
unsigned l = 0; l != NumElts; l += 8) {
3288 for (
unsigned i = 0; i != 4; ++i)
3289 Idxs[i + l] = i + l;
3290 for (
unsigned i = 0; i != 4; ++i)
3291 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3299 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.shuf.p")) {
3302 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3303 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3306 unsigned HalfLaneElts = NumLaneElts / 2;
3309 for (
unsigned i = 0; i != NumElts; ++i) {
3311 Idxs[i] = i - (i % NumLaneElts);
3313 if ((i % NumLaneElts) >= HalfLaneElts)
3317 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3324 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.movddup") ||
3325 Name.starts_with(
"avx512.mask.movshdup") ||
3326 Name.starts_with(
"avx512.mask.movsldup"))) {
3328 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3332 if (
Name.starts_with(
"avx512.mask.movshdup."))
3336 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
3337 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
3338 Idxs[i + l + 0] = i + l +
Offset;
3339 Idxs[i + l + 1] = i + l +
Offset;
3346 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckl") ||
3347 Name.starts_with(
"avx512.mask.unpckl."))) {
3350 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3354 for (
int l = 0; l != NumElts; l += NumLaneElts)
3355 for (
int i = 0; i != NumLaneElts; ++i)
3356 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3362 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckh") ||
3363 Name.starts_with(
"avx512.mask.unpckh."))) {
3366 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3370 for (
int l = 0; l != NumElts; l += NumLaneElts)
3371 for (
int i = 0; i != NumLaneElts; ++i)
3372 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3378 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.and.") ||
3379 Name.starts_with(
"avx512.mask.pand."))) {
3381 VectorType *ITy = VectorType::getInteger(FTy);
3387 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.andn.") ||
3388 Name.starts_with(
"avx512.mask.pandn."))) {
3390 VectorType *ITy = VectorType::getInteger(FTy);
3397 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.or.") ||
3398 Name.starts_with(
"avx512.mask.por."))) {
3400 VectorType *ITy = VectorType::getInteger(FTy);
3406 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.xor.") ||
3407 Name.starts_with(
"avx512.mask.pxor."))) {
3409 VectorType *ITy = VectorType::getInteger(FTy);
3415 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.padd.")) {
3419 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psub.")) {
3423 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pmull.")) {
3427 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.add.p")) {
3428 if (
Name.ends_with(
".512")) {
3430 if (
Name[17] ==
's')
3431 IID = Intrinsic::x86_avx512_add_ps_512;
3433 IID = Intrinsic::x86_avx512_add_pd_512;
3436 { CI->getArgOperand(0), CI->getArgOperand(1),
3437 CI->getArgOperand(4) });
3443 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.div.p")) {
3444 if (
Name.ends_with(
".512")) {
3446 if (
Name[17] ==
's')
3447 IID = Intrinsic::x86_avx512_div_ps_512;
3449 IID = Intrinsic::x86_avx512_div_pd_512;
3452 { CI->getArgOperand(0), CI->getArgOperand(1),
3453 CI->getArgOperand(4) });
3459 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.mul.p")) {
3460 if (
Name.ends_with(
".512")) {
3462 if (
Name[17] ==
's')
3463 IID = Intrinsic::x86_avx512_mul_ps_512;
3465 IID = Intrinsic::x86_avx512_mul_pd_512;
3468 { CI->getArgOperand(0), CI->getArgOperand(1),
3469 CI->getArgOperand(4) });
3475 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.sub.p")) {
3476 if (
Name.ends_with(
".512")) {
3478 if (
Name[17] ==
's')
3479 IID = Intrinsic::x86_avx512_sub_ps_512;
3481 IID = Intrinsic::x86_avx512_sub_pd_512;
3484 { CI->getArgOperand(0), CI->getArgOperand(1),
3485 CI->getArgOperand(4) });
3491 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.max.p") ||
3492 Name.starts_with(
"avx512.mask.min.p")) &&
3493 Name.drop_front(18) ==
".512") {
3494 bool IsDouble =
Name[17] ==
'd';
3495 bool IsMin =
Name[13] ==
'i';
3497 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3498 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3503 { CI->getArgOperand(0), CI->getArgOperand(1),
3504 CI->getArgOperand(4) });
3507 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.lzcnt.")) {
3511 { CI->getArgOperand(0), Builder.getInt1(false) });
3514 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psll")) {
3515 bool IsImmediate =
Name[16] ==
'i' ||
3516 (
Name.size() > 18 &&
Name[18] ==
'i');
3517 bool IsVariable =
Name[16] ==
'v';
3524 if (IsVariable &&
Name[17] !=
'.') {
3525 if (
Size ==
'd' &&
Name[17] ==
'2')
3526 IID = Intrinsic::x86_avx2_psllv_q;
3527 else if (
Size ==
'd' &&
Name[17] ==
'4')
3528 IID = Intrinsic::x86_avx2_psllv_q_256;
3529 else if (
Size ==
's' &&
Name[17] ==
'4')
3530 IID = Intrinsic::x86_avx2_psllv_d;
3531 else if (
Size ==
's' &&
Name[17] ==
'8')
3532 IID = Intrinsic::x86_avx2_psllv_d_256;
3533 else if (
Size ==
'h' &&
Name[17] ==
'8')
3534 IID = Intrinsic::x86_avx512_psllv_w_128;
3535 else if (
Size ==
'h' &&
Name[17] ==
'1')
3536 IID = Intrinsic::x86_avx512_psllv_w_256;
3537 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3538 IID = Intrinsic::x86_avx512_psllv_w_512;
3541 }
else if (
Name.ends_with(
".128")) {
3543 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3544 : Intrinsic::x86_sse2_psll_d;
3545 else if (
Size ==
'q')
3546 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3547 : Intrinsic::x86_sse2_psll_q;
3548 else if (
Size ==
'w')
3549 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3550 : Intrinsic::x86_sse2_psll_w;
3553 }
else if (
Name.ends_with(
".256")) {
3555 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3556 : Intrinsic::x86_avx2_psll_d;
3557 else if (
Size ==
'q')
3558 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3559 : Intrinsic::x86_avx2_psll_q;
3560 else if (
Size ==
'w')
3561 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3562 : Intrinsic::x86_avx2_psll_w;
3567 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3568 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3569 Intrinsic::x86_avx512_psll_d_512;
3570 else if (
Size ==
'q')
3571 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3572 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3573 Intrinsic::x86_avx512_psll_q_512;
3574 else if (
Size ==
'w')
3575 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3576 : Intrinsic::x86_avx512_psll_w_512;
3582 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psrl")) {
3583 bool IsImmediate =
Name[16] ==
'i' ||
3584 (
Name.size() > 18 &&
Name[18] ==
'i');
3585 bool IsVariable =
Name[16] ==
'v';
3592 if (IsVariable &&
Name[17] !=
'.') {
3593 if (
Size ==
'd' &&
Name[17] ==
'2')
3594 IID = Intrinsic::x86_avx2_psrlv_q;
3595 else if (
Size ==
'd' &&
Name[17] ==
'4')
3596 IID = Intrinsic::x86_avx2_psrlv_q_256;
3597 else if (
Size ==
's' &&
Name[17] ==
'4')
3598 IID = Intrinsic::x86_avx2_psrlv_d;
3599 else if (
Size ==
's' &&
Name[17] ==
'8')
3600 IID = Intrinsic::x86_avx2_psrlv_d_256;
3601 else if (
Size ==
'h' &&
Name[17] ==
'8')
3602 IID = Intrinsic::x86_avx512_psrlv_w_128;
3603 else if (
Size ==
'h' &&
Name[17] ==
'1')
3604 IID = Intrinsic::x86_avx512_psrlv_w_256;
3605 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3606 IID = Intrinsic::x86_avx512_psrlv_w_512;
3609 }
else if (
Name.ends_with(
".128")) {
3611 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3612 : Intrinsic::x86_sse2_psrl_d;
3613 else if (
Size ==
'q')
3614 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3615 : Intrinsic::x86_sse2_psrl_q;
3616 else if (
Size ==
'w')
3617 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3618 : Intrinsic::x86_sse2_psrl_w;
3621 }
else if (
Name.ends_with(
".256")) {
3623 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3624 : Intrinsic::x86_avx2_psrl_d;
3625 else if (
Size ==
'q')
3626 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3627 : Intrinsic::x86_avx2_psrl_q;
3628 else if (
Size ==
'w')
3629 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3630 : Intrinsic::x86_avx2_psrl_w;
3635 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3636 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3637 Intrinsic::x86_avx512_psrl_d_512;
3638 else if (
Size ==
'q')
3639 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3640 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3641 Intrinsic::x86_avx512_psrl_q_512;
3642 else if (
Size ==
'w')
3643 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3644 : Intrinsic::x86_avx512_psrl_w_512;
3650 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psra")) {
3651 bool IsImmediate =
Name[16] ==
'i' ||
3652 (
Name.size() > 18 &&
Name[18] ==
'i');
3653 bool IsVariable =
Name[16] ==
'v';
3660 if (IsVariable &&
Name[17] !=
'.') {
3661 if (
Size ==
's' &&
Name[17] ==
'4')
3662 IID = Intrinsic::x86_avx2_psrav_d;
3663 else if (
Size ==
's' &&
Name[17] ==
'8')
3664 IID = Intrinsic::x86_avx2_psrav_d_256;
3665 else if (
Size ==
'h' &&
Name[17] ==
'8')
3666 IID = Intrinsic::x86_avx512_psrav_w_128;
3667 else if (
Size ==
'h' &&
Name[17] ==
'1')
3668 IID = Intrinsic::x86_avx512_psrav_w_256;
3669 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3670 IID = Intrinsic::x86_avx512_psrav_w_512;
3673 }
else if (
Name.ends_with(
".128")) {
3675 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3676 : Intrinsic::x86_sse2_psra_d;
3677 else if (
Size ==
'q')
3678 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3679 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3680 Intrinsic::x86_avx512_psra_q_128;
3681 else if (
Size ==
'w')
3682 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3683 : Intrinsic::x86_sse2_psra_w;
3686 }
else if (
Name.ends_with(
".256")) {
3688 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3689 : Intrinsic::x86_avx2_psra_d;
3690 else if (
Size ==
'q')
3691 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3692 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3693 Intrinsic::x86_avx512_psra_q_256;
3694 else if (
Size ==
'w')
3695 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3696 : Intrinsic::x86_avx2_psra_w;
3701 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3702 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3703 Intrinsic::x86_avx512_psra_d_512;
3704 else if (
Size ==
'q')
3705 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3706 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3707 Intrinsic::x86_avx512_psra_q_512;
3708 else if (
Size ==
'w')
3709 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3710 : Intrinsic::x86_avx512_psra_w_512;
3716 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.move.s")) {
3718 }
else if (IsX86 &&
Name.starts_with(
"avx512.cvtmask2")) {
3720 }
else if (IsX86 &&
Name.ends_with(
".movntdqa")) {
3728 Ptr, PointerType::getUnqual(CI->
getType()),
"cast");
3732 LI->
setMetadata(LLVMContext::MD_nontemporal, Node);
3734 }
else if (IsX86 && (
Name.starts_with(
"fma.vfmadd.") ||
3735 Name.starts_with(
"fma.vfmsub.") ||
3736 Name.starts_with(
"fma.vfnmadd.") ||
3737 Name.starts_with(
"fma.vfnmsub."))) {
3738 bool NegMul =
Name[6] ==
'n';
3739 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3740 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3751 if (NegMul && !IsScalar)
3753 if (NegMul && IsScalar)
3766 }
else if (IsX86 &&
Name.starts_with(
"fma4.vfmadd.s")) {
3781 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.s") ||
3782 Name.starts_with(
"avx512.maskz.vfmadd.s") ||
3783 Name.starts_with(
"avx512.mask3.vfmadd.s") ||
3784 Name.starts_with(
"avx512.mask3.vfmsub.s") ||
3785 Name.starts_with(
"avx512.mask3.vfnmsub.s"))) {
3786 bool IsMask3 =
Name[11] ==
'3';
3787 bool IsMaskZ =
Name[11] ==
'z';
3789 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3790 bool NegMul =
Name[2] ==
'n';
3791 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3797 if (NegMul && (IsMask3 || IsMaskZ))
3799 if (NegMul && !(IsMask3 || IsMaskZ))
3809 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3813 if (
Name.back() ==
'd')
3814 IID = Intrinsic::x86_avx512_vfmadd_f64;
3816 IID = Intrinsic::x86_avx512_vfmadd_f32;
3831 if (NegAcc && IsMask3)
3838 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.p") ||
3839 Name.starts_with(
"avx512.mask.vfnmadd.p") ||
3840 Name.starts_with(
"avx512.mask.vfnmsub.p") ||
3841 Name.starts_with(
"avx512.mask3.vfmadd.p") ||
3842 Name.starts_with(
"avx512.mask3.vfmsub.p") ||
3843 Name.starts_with(
"avx512.mask3.vfnmsub.p") ||
3844 Name.starts_with(
"avx512.maskz.vfmadd.p"))) {
3845 bool IsMask3 =
Name[11] ==
'3';
3846 bool IsMaskZ =
Name[11] ==
'z';
3848 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3849 bool NegMul =
Name[2] ==
'n';
3850 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3856 if (NegMul && (IsMask3 || IsMaskZ))
3858 if (NegMul && !(IsMask3 || IsMaskZ))
3865 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
3869 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3871 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3874 { A, B, C, CI->getArgOperand(4) });
3887 }
else if (IsX86 &&
Name.starts_with(
"fma.vfmsubadd.p")) {
3891 if (VecWidth == 128 && EltWidth == 32)
3892 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3893 else if (VecWidth == 256 && EltWidth == 32)
3894 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3895 else if (VecWidth == 128 && EltWidth == 64)
3896 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3897 else if (VecWidth == 256 && EltWidth == 64)
3898 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3907 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmaddsub.p") ||
3908 Name.starts_with(
"avx512.mask3.vfmaddsub.p") ||
3909 Name.starts_with(
"avx512.maskz.vfmaddsub.p") ||
3910 Name.starts_with(
"avx512.mask3.vfmsubadd.p"))) {
3911 bool IsMask3 =
Name[11] ==
'3';
3912 bool IsMaskZ =
Name[11] ==
'z';
3914 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3915 bool IsSubAdd =
Name[3] ==
's';
3920 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3922 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3932 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3947 for (
int i = 0; i != NumElts; ++i)
3948 Idxs[i] = i + (i % 2) * NumElts;
3958 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pternlog.") ||
3959 Name.starts_with(
"avx512.maskz.pternlog."))) {
3960 bool ZeroMask =
Name[11] ==
'z';
3964 if (VecWidth == 128 && EltWidth == 32)
3965 IID = Intrinsic::x86_avx512_pternlog_d_128;
3966 else if (VecWidth == 256 && EltWidth == 32)
3967 IID = Intrinsic::x86_avx512_pternlog_d_256;
3968 else if (VecWidth == 512 && EltWidth == 32)
3969 IID = Intrinsic::x86_avx512_pternlog_d_512;
3970 else if (VecWidth == 128 && EltWidth == 64)
3971 IID = Intrinsic::x86_avx512_pternlog_q_128;
3972 else if (VecWidth == 256 && EltWidth == 64)
3973 IID = Intrinsic::x86_avx512_pternlog_q_256;
3974 else if (VecWidth == 512 && EltWidth == 64)
3975 IID = Intrinsic::x86_avx512_pternlog_q_512;
3986 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpmadd52") ||
3987 Name.starts_with(
"avx512.maskz.vpmadd52"))) {
3988 bool ZeroMask =
Name[11] ==
'z';
3992 if (VecWidth == 128 && !
High)
3993 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3994 else if (VecWidth == 256 && !
High)
3995 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3996 else if (VecWidth == 512 && !
High)
3997 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3998 else if (VecWidth == 128 &&
High)
3999 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4000 else if (VecWidth == 256 &&
High)
4001 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4002 else if (VecWidth == 512 &&
High)
4003 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4014 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpermi2var.") ||
4015 Name.starts_with(
"avx512.mask.vpermt2var.") ||
4016 Name.starts_with(
"avx512.maskz.vpermt2var."))) {
4017 bool ZeroMask =
Name[11] ==
'z';
4018 bool IndexForm =
Name[17] ==
'i';
4020 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpbusd.") ||
4021 Name.starts_with(
"avx512.maskz.vpdpbusd.") ||
4022 Name.starts_with(
"avx512.mask.vpdpbusds.") ||
4023 Name.starts_with(
"avx512.maskz.vpdpbusds."))) {
4024 bool ZeroMask =
Name[11] ==
'z';
4025 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4028 if (VecWidth == 128 && !IsSaturating)
4029 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4030 else if (VecWidth == 256 && !IsSaturating)
4031 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4032 else if (VecWidth == 512 && !IsSaturating)
4033 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4034 else if (VecWidth == 128 && IsSaturating)
4035 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4036 else if (VecWidth == 256 && IsSaturating)
4037 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4038 else if (VecWidth == 512 && IsSaturating)
4039 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4050 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpwssd.") ||
4051 Name.starts_with(
"avx512.maskz.vpdpwssd.") ||
4052 Name.starts_with(
"avx512.mask.vpdpwssds.") ||
4053 Name.starts_with(
"avx512.maskz.vpdpwssds."))) {
4054 bool ZeroMask =
Name[11] ==
'z';
4055 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4058 if (VecWidth == 128 && !IsSaturating)
4059 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4060 else if (VecWidth == 256 && !IsSaturating)
4061 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4062 else if (VecWidth == 512 && !IsSaturating)
4063 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4064 else if (VecWidth == 128 && IsSaturating)
4065 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4066 else if (VecWidth == 256 && IsSaturating)
4067 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4068 else if (VecWidth == 512 && IsSaturating)
4069 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4080 }
else if (IsX86 && (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
4081 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
4082 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64")) {
4084 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
4085 IID = Intrinsic::x86_addcarry_32;
4086 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
4087 IID = Intrinsic::x86_addcarry_64;
4088 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
4089 IID = Intrinsic::x86_subborrow_32;
4090 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
4091 IID = Intrinsic::x86_subborrow_64;
4113 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.") &&
4116 }
else if (IsNVVM && (
Name ==
"abs.i" ||
Name ==
"abs.ll")) {
4122 }
else if (IsNVVM && (
Name.starts_with(
"atomic.load.add.f32.p") ||
4123 Name.starts_with(
"atomic.load.add.f64.p"))) {
4127 AtomicOrdering::SequentiallyConsistent);
4128 }
else if (IsNVVM &&
Name.consume_front(
"max.") &&
4137 }
else if (IsNVVM &&
Name.consume_front(
"min.") &&
4146 }
else if (IsNVVM &&
Name ==
"clz.ll") {
4152 {Arg, Builder.getFalse()},
"ctlz");
4154 }
else if (IsNVVM &&
Name ==
"popc.ll") {
4163 }
else if (IsNVVM) {
4164 if (
Name ==
"h2f") {
4167 F->getParent(), Intrinsic::convert_from_fp16,
4168 {Builder.getFloatTy()}),
4173 !
F->getReturnType()->getScalarType()->isBFloatTy()) {
4187 if (
F->getReturnType()->isIntegerTy())
4193 }
else if (IsAMDGCN) {
4205 const auto &DefaultCase = [&]() ->
void {
4210 "Unknown function for CallBase upgrade and isn't just a name change");
4216 if (
auto *OldST = dyn_cast<StructType>(CI->
getType())) {
4218 "Return type must have changed");
4219 assert(OldST->getNumElements() ==
4220 cast<StructType>(NewFn->
getReturnType())->getNumElements() &&
4221 "Must have same number of elements");
4226 for (
unsigned Idx = 0;
Idx < OldST->getNumElements(); ++
Idx) {
4247 case Intrinsic::arm_neon_vst1:
4248 case Intrinsic::arm_neon_vst2:
4249 case Intrinsic::arm_neon_vst3:
4250 case Intrinsic::arm_neon_vst4:
4251 case Intrinsic::arm_neon_vst2lane:
4252 case Intrinsic::arm_neon_vst3lane:
4253 case Intrinsic::arm_neon_vst4lane: {
4258 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4259 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4260 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4264 cast<ConstantInt>(Args[3])->getZExtValue());
4268 case Intrinsic::aarch64_sve_ld3_sret:
4269 case Intrinsic::aarch64_sve_ld4_sret:
4270 case Intrinsic::aarch64_sve_ld2_sret: {
4279 dyn_cast<ScalableVectorType>(
F->getReturnType());
4280 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4284 for (
unsigned I = 0;
I <
N;
I++) {
4289 NewCall = dyn_cast<CallInst>(Ret);
4293 case Intrinsic::coro_end: {
4300 case Intrinsic::vector_extract: {
4303 if (!
Name.starts_with(
"aarch64.sve.tuple.get")) {
4308 dyn_cast<ScalableVectorType>(
F->getReturnType());
4309 unsigned MinElts =
RetTy->getMinNumElements();
4310 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4316 case Intrinsic::vector_insert: {
4319 if (!
Name.starts_with(
"aarch64.sve.tuple")) {
4323 if (
Name.starts_with(
"aarch64.sve.tuple.set")) {
4324 unsigned I = dyn_cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4333 if (
Name.starts_with(
"aarch64.sve.tuple.create")) {
4339 assert(
N > 1 &&
"Create is expected to be between 2-4");
4341 dyn_cast<ScalableVectorType>(
F->getReturnType());
4343 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4344 for (
unsigned I = 0;
I <
N;
I++) {
4349 NewCall = dyn_cast<CallInst>(Ret);
4354 case Intrinsic::arm_neon_bfdot:
4355 case Intrinsic::arm_neon_bfmmla:
4356 case Intrinsic::arm_neon_bfmlalb:
4357 case Intrinsic::arm_neon_bfmlalt:
4358 case Intrinsic::aarch64_neon_bfdot:
4359 case Intrinsic::aarch64_neon_bfmmla:
4360 case Intrinsic::aarch64_neon_bfmlalb:
4361 case Intrinsic::aarch64_neon_bfmlalt: {
4364 "Mismatch between function args and call args");
4365 size_t OperandWidth =
4367 assert((OperandWidth == 64 || OperandWidth == 128) &&
4368 "Unexpected operand width");
4370 auto Iter = CI->
args().begin();
4371 Args.push_back(*Iter++);
4378 case Intrinsic::bitreverse:
4382 case Intrinsic::ctlz:
4383 case Intrinsic::cttz:
4385 "Mismatch between function args and call args");
4390 case Intrinsic::objectsize: {
4391 Value *NullIsUnknownSize =
4400 case Intrinsic::ctpop:
4404 case Intrinsic::convert_from_fp16:
4408 case Intrinsic::dbg_value: {
4412 if (
Name.starts_with(
"dbg.addr")) {
4414 cast<MetadataAsValue>(CI->
getArgOperand(2))->getMetadata());
4426 if (
Offset->isZeroValue()) {
4436 case Intrinsic::ptr_annotation:
4453 case Intrinsic::var_annotation:
4469 case Intrinsic::riscv_aes32dsi:
4470 case Intrinsic::riscv_aes32dsmi:
4471 case Intrinsic::riscv_aes32esi:
4472 case Intrinsic::riscv_aes32esmi:
4473 case Intrinsic::riscv_sm4ks:
4474 case Intrinsic::riscv_sm4ed: {
4489 cast<ConstantInt>(Arg2)->getZExtValue());
4491 NewCall = Builder.
CreateCall(NewFn, {Arg0, Arg1, Arg2});
4492 Value *Res = NewCall;
4500 case Intrinsic::riscv_sha256sig0:
4501 case Intrinsic::riscv_sha256sig1:
4502 case Intrinsic::riscv_sha256sum0:
4503 case Intrinsic::riscv_sha256sum1:
4504 case Intrinsic::riscv_sm3p0:
4505 case Intrinsic::riscv_sm3p1: {
4523 case Intrinsic::x86_xop_vfrcz_ss:
4524 case Intrinsic::x86_xop_vfrcz_sd:
4528 case Intrinsic::x86_xop_vpermil2pd:
4529 case Intrinsic::x86_xop_vpermil2ps:
4530 case Intrinsic::x86_xop_vpermil2pd_256:
4531 case Intrinsic::x86_xop_vpermil2ps_256: {
4534 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4540 case Intrinsic::x86_sse41_ptestc:
4541 case Intrinsic::x86_sse41_ptestz:
4542 case Intrinsic::x86_sse41_ptestnzc: {
4559 NewCall = Builder.
CreateCall(NewFn, {BC0, BC1});
4563 case Intrinsic::x86_rdtscp: {
4585 case Intrinsic::x86_sse41_insertps:
4586 case Intrinsic::x86_sse41_dppd:
4587 case Intrinsic::x86_sse41_dpps:
4588 case Intrinsic::x86_sse41_mpsadbw:
4589 case Intrinsic::x86_avx_dp_ps_256:
4590 case Intrinsic::x86_avx2_mpsadbw: {
4601 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4602 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4603 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4604 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4605 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4606 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4609 cast<FixedVectorType>(Args[0]->
getType())->getNumElements();
4621 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4622 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4623 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4624 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4625 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4626 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4628 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4630 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4643 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4644 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4645 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4648 cast<FixedVectorType>(CI->
getType())->getNumElements() * 2;
4658 case Intrinsic::thread_pointer: {
4663 case Intrinsic::memcpy:
4664 case Intrinsic::memmove:
4665 case Intrinsic::memset: {
4685 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4686 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4688 auto *MemCI = cast<MemIntrinsic>(NewCall);
4691 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
4693 if (
auto *MTI = dyn_cast<MemTransferInst>(MemCI))
4694 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
4698 assert(NewCall &&
"Should have either set this variable or returned through "
4699 "the default case");
4706 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
4715 if (
CallBase *CB = dyn_cast<CallBase>(U))
4719 F->eraseFromParent();
4725 if (NumOperands == 0)
4729 if (isa<MDNode>(MD.
getOperand(0)) && NumOperands >= 3)
4733 if (NumOperands == 3) {
4737 Metadata *Elts2[] = {ScalarType, ScalarType,
4751 if (Opc != Instruction::BitCast)
4755 Type *SrcTy = V->getType();
4772 if (Opc != Instruction::BitCast)
4775 Type *SrcTy =
C->getType();
4799 bool BrokenDebugInfo =
false;
4802 if (!BrokenDebugInfo)
4808 M.getContext().diagnose(Diag);
4815 M.getContext().diagnose(DiagVersion);
4823 bool Changed =
false;
4824 const char *MarkerKey =
"clang.arc.retainAutoreleasedReturnValueMarker";
4825 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4826 if (ModRetainReleaseMarker) {
4829 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(0));
4832 ID->getString().split(ValueComp,
"#");
4833 if (ValueComp.
size() == 2) {
4834 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
4838 M.eraseNamedMetadata(ModRetainReleaseMarker);
4849 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
4851 Function *Fn = M.getFunction(OldFunc);
4859 CallInst *CI = dyn_cast<CallInst>(U);
4874 bool InvalidCast =
false;
4881 if (I < NewFuncTy->getNumParams()) {
4891 Args.push_back(Arg);
4916 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4924 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4925 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4926 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4927 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4928 {
"objc_autoreleaseReturnValue",
4929 llvm::Intrinsic::objc_autoreleaseReturnValue},
4930 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4931 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4932 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4933 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4934 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4935 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4936 {
"objc_release", llvm::Intrinsic::objc_release},
4937 {
"objc_retain", llvm::Intrinsic::objc_retain},
4938 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4939 {
"objc_retainAutoreleaseReturnValue",
4940 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4941 {
"objc_retainAutoreleasedReturnValue",
4942 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4943 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4944 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4945 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4946 {
"objc_unsafeClaimAutoreleasedReturnValue",
4947 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4948 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4949 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4950 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4951 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4952 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4953 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4954 {
"objc_arc_annotation_topdown_bbstart",
4955 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4956 {
"objc_arc_annotation_topdown_bbend",
4957 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4958 {
"objc_arc_annotation_bottomup_bbstart",
4959 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4960 {
"objc_arc_annotation_bottomup_bbend",
4961 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4963 for (
auto &
I : RuntimeFuncs)
4964 UpgradeToIntrinsic(
I.first,
I.second);
4968 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
4972 bool HasObjCFlag =
false, HasClassProperties =
false, Changed =
false;
4973 bool HasSwiftVersionFlag =
false;
4974 uint8_t SwiftMajorVersion, SwiftMinorVersion;
4983 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(1));
4995 if (
ID->getString() ==
"Objective-C Image Info Version")
4997 if (
ID->getString() ==
"Objective-C Class Properties")
4998 HasClassProperties =
true;
5000 if (
ID->getString() ==
"PIC Level") {
5001 if (
auto *Behavior =
5002 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5003 uint64_t V = Behavior->getLimitedValue();
5009 if (
ID->getString() ==
"PIE Level")
5010 if (
auto *Behavior =
5011 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0)))
5017 if (
ID->getString() ==
"branch-target-enforcement" ||
5018 ID->getString().starts_with(
"sign-return-address")) {
5019 if (
auto *Behavior =
5020 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5025 Op->getOperand(1),
Op->getOperand(2)};
5035 if (
ID->getString() ==
"Objective-C Image Info Section") {
5036 if (
auto *
Value = dyn_cast_or_null<MDString>(
Op->getOperand(2))) {
5038 Value->getString().split(ValueComp,
" ");
5039 if (ValueComp.
size() != 1) {
5040 std::string NewValue;
5041 for (
auto &S : ValueComp)
5042 NewValue += S.str();
5043 Metadata *Ops[3] = {
Op->getOperand(0),
Op->getOperand(1),
5053 if (
ID->getString() ==
"Objective-C Garbage Collection") {
5054 auto Md = dyn_cast<ConstantAsMetadata>(
Op->getOperand(2));
5056 assert(Md->getValue() &&
"Expected non-empty metadata");
5057 auto Type = Md->getValue()->getType();
5060 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
5061 if ((Val & 0xff) != Val) {
5062 HasSwiftVersionFlag =
true;
5063 SwiftABIVersion = (Val & 0xff00) >> 8;
5064 SwiftMajorVersion = (Val & 0xff000000) >> 24;
5065 SwiftMinorVersion = (Val & 0xff0000) >> 16;
5076 if (
ID->getString() ==
"amdgpu_code_object_version") {
5079 MDString::get(M.getContext(),
"amdhsa_code_object_version"),
5091 if (HasObjCFlag && !HasClassProperties) {
5097 if (HasSwiftVersionFlag) {
5101 ConstantInt::get(Int8Ty, SwiftMajorVersion));
5103 ConstantInt::get(Int8Ty, SwiftMinorVersion));
5111 auto TrimSpaces = [](
StringRef Section) -> std::string {
5113 Section.split(Components,
',');
5118 for (
auto Component : Components)
5119 OS <<
',' << Component.trim();
5121 return std::string(
OS.str().substr(1));
5124 for (
auto &GV : M.globals()) {
5125 if (!GV.hasSection())
5130 if (!Section.starts_with(
"__DATA, __objc_catlist"))
5135 GV.setSection(TrimSpaces(Section));
5151struct StrictFPUpgradeVisitor :
public InstVisitor<StrictFPUpgradeVisitor> {
5152 StrictFPUpgradeVisitor() =
default;
5155 if (!Call.isStrictFP())
5157 if (isa<ConstrainedFPIntrinsic>(&Call))
5161 Call.removeFnAttr(Attribute::StrictFP);
5162 Call.addFnAttr(Attribute::NoBuiltin);
5170 if (!
F.isDeclaration() && !
F.hasFnAttribute(Attribute::StrictFP)) {
5171 StrictFPUpgradeVisitor SFPV;
5177 for (
auto &Arg :
F.args())
5182 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5185 if (
T->getNumOperands() < 1)
5187 auto *S = dyn_cast_or_null<MDString>(
T->getOperand(0));
5190 return S->getString().starts_with(
"llvm.vectorizer.");
5194 StringRef OldPrefix =
"llvm.vectorizer.";
5197 if (OldTag ==
"llvm.vectorizer.unroll")
5206 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5209 if (
T->getNumOperands() < 1)
5211 auto *OldTag = dyn_cast_or_null<MDString>(
T->getOperand(0));
5214 if (!OldTag->getString().starts_with(
"llvm.vectorizer."))
5221 for (
unsigned I = 1,
E =
T->getNumOperands();
I !=
E; ++
I)
5228 auto *
T = dyn_cast<MDTuple>(&
N);
5247 if (
T.isAMDGPU() && !
T.isAMDGCN() && !
DL.contains(
"-G") &&
5248 !
DL.starts_with(
"G")) {
5249 return DL.
empty() ? std::string(
"G1") : (
DL +
"-G1").str();
5252 if (
T.isRISCV64()) {
5254 auto I =
DL.find(
"-n64-");
5256 return (
DL.take_front(
I) +
"-n32:64-" +
DL.drop_front(
I + 5)).str();
5260 std::string Res =
DL.str();
5264 if (!
DL.contains(
"-G") && !
DL.starts_with(
"G"))
5265 Res.append(Res.empty() ?
"G1" :
"-G1");
5270 if (!
DL.contains(
"-ni") && !
DL.starts_with(
"ni"))
5271 Res.append(
"-ni:7:8:9");
5273 if (
DL.ends_with(
"ni:7"))
5275 if (
DL.ends_with(
"ni:7:8"))
5280 if (!
DL.contains(
"-p7") && !
DL.starts_with(
"p7"))
5281 Res.append(
"-p7:160:256:256:32");
5282 if (!
DL.contains(
"-p8") && !
DL.starts_with(
"p8"))
5283 Res.append(
"-p8:128:128");
5284 if (!
DL.contains(
"-p9") && !
DL.starts_with(
"p9"))
5285 Res.append(
"-p9:192:256:256:32");
5295 std::string AddrSpaces =
"-p270:32:32-p271:32:32-p272:64:64";
5298 Regex R(
"(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
5299 if (R.match(Res, &
Groups))
5309 if (!
T.isOSIAMCU()) {
5310 std::string I128 =
"-i128:128";
5313 Regex R(
"^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
5314 if (R.match(Res, &
Groups))
5322 if (
T.isWindowsMSVCEnvironment() && !
T.isArch64Bit()) {
5324 auto I =
Ref.find(
"-f80:32-");
5326 Res = (
Ref.take_front(
I) +
"-f80:128-" +
Ref.drop_front(
I + 8)).str();
5334 Attribute A =
B.getAttribute(
"no-frame-pointer-elim");
5337 FramePointer =
A.getValueAsString() ==
"true" ?
"all" :
"none";
5338 B.removeAttribute(
"no-frame-pointer-elim");
5340 if (
B.contains(
"no-frame-pointer-elim-non-leaf")) {
5342 if (FramePointer !=
"all")
5343 FramePointer =
"non-leaf";
5344 B.removeAttribute(
"no-frame-pointer-elim-non-leaf");
5346 if (!FramePointer.
empty())
5347 B.addAttribute(
"frame-pointer", FramePointer);
5349 A =
B.getAttribute(
"null-pointer-is-valid");
5352 bool NullPointerIsValid =
A.getValueAsString() ==
"true";
5353 B.removeAttribute(
"null-pointer-is-valid");
5354 if (NullPointerIsValid)
5355 B.addAttribute(Attribute::NullPointerIsValid);
5365 return OBD.
getTag() ==
"clang.arc.attachedcall" &&
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Metadata * upgradeLoopArgument(Metadata *MD)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ UIncWrap
Increment one up to a maximum value.
@ UDecWrap
Decrement one until a minimum value or zero.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name, BasicBlock::iterator InsertBefore)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static ConstantAggregateZero * get(Type *Ty)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
LinkageTypes getLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Type * getBFloatTy()
Fetch the type representing a 16-bit brain floating point value.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
RetTy visitCallBase(CallBase &I)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVMContext & getContext() const
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
void setOperand(unsigned I, MDNode *New)
MDNode * getOperand(unsigned i) const
unsigned getNumOperands() const
A container for an operand bundle being viewed as a set of values rather than a set of uses.
ArrayRef< InputTy > inputs() const
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Class to represent scalable SIMD vectors.
uint64_t getMinNumElements() const
Get the minimum number of elements in this vector.
ArrayRef< int > getShuffleMask() const
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getBFloatTy(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
This is an optimization pass for GlobalISel generic memory operations.
void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn)
This is a more granular function that simply checks an intrinsic function for upgrading,...
MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
unsigned getDebugMetadataVersionFromModule(const Module &M)
Return Debug Info Metadata Version by checking module flags.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
@ Dynamic
Denotes mode unknown at compile time.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.