30#include "llvm/IR/IntrinsicsAArch64.h"
31#include "llvm/IR/IntrinsicsARM.h"
32#include "llvm/IR/IntrinsicsNVPTX.h"
33#include "llvm/IR/IntrinsicsRISCV.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/IR/IntrinsicsX86.h"
53 cl::desc(
"Disable autoupgrade of debug info"));
63 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
78 Type *LastArgType =
F->getFunctionType()->getParamType(
79 F->getFunctionType()->getNumParams() - 1);
94 if (
F->getReturnType()->isVectorTy())
104 if (
F->getReturnType()->getScalarType()->isBFloatTy())
114 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
128 if (
Name.consume_front(
"avx."))
129 return (
Name.starts_with(
"blend.p") ||
130 Name ==
"cvt.ps2.pd.256" ||
131 Name ==
"cvtdq2.pd.256" ||
132 Name ==
"cvtdq2.ps.256" ||
133 Name.starts_with(
"movnt.") ||
134 Name.starts_with(
"sqrt.p") ||
135 Name.starts_with(
"storeu.") ||
136 Name.starts_with(
"vbroadcast.s") ||
137 Name.starts_with(
"vbroadcastf128") ||
138 Name.starts_with(
"vextractf128.") ||
139 Name.starts_with(
"vinsertf128.") ||
140 Name.starts_with(
"vperm2f128.") ||
141 Name.starts_with(
"vpermil."));
143 if (
Name.consume_front(
"avx2."))
144 return (
Name ==
"movntdqa" ||
145 Name.starts_with(
"pabs.") ||
146 Name.starts_with(
"padds.") ||
147 Name.starts_with(
"paddus.") ||
148 Name.starts_with(
"pblendd.") ||
150 Name.starts_with(
"pbroadcast") ||
151 Name.starts_with(
"pcmpeq.") ||
152 Name.starts_with(
"pcmpgt.") ||
153 Name.starts_with(
"pmax") ||
154 Name.starts_with(
"pmin") ||
155 Name.starts_with(
"pmovsx") ||
156 Name.starts_with(
"pmovzx") ||
158 Name ==
"pmulu.dq" ||
159 Name.starts_with(
"psll.dq") ||
160 Name.starts_with(
"psrl.dq") ||
161 Name.starts_with(
"psubs.") ||
162 Name.starts_with(
"psubus.") ||
163 Name.starts_with(
"vbroadcast") ||
164 Name ==
"vbroadcasti128" ||
165 Name ==
"vextracti128" ||
166 Name ==
"vinserti128" ||
167 Name ==
"vperm2i128");
169 if (
Name.consume_front(
"avx512.")) {
170 if (
Name.consume_front(
"mask."))
172 return (
Name.starts_with(
"add.p") ||
173 Name.starts_with(
"and.") ||
174 Name.starts_with(
"andn.") ||
175 Name.starts_with(
"broadcast.s") ||
176 Name.starts_with(
"broadcastf32x4.") ||
177 Name.starts_with(
"broadcastf32x8.") ||
178 Name.starts_with(
"broadcastf64x2.") ||
179 Name.starts_with(
"broadcastf64x4.") ||
180 Name.starts_with(
"broadcasti32x4.") ||
181 Name.starts_with(
"broadcasti32x8.") ||
182 Name.starts_with(
"broadcasti64x2.") ||
183 Name.starts_with(
"broadcasti64x4.") ||
184 Name.starts_with(
"cmp.b") ||
185 Name.starts_with(
"cmp.d") ||
186 Name.starts_with(
"cmp.q") ||
187 Name.starts_with(
"cmp.w") ||
188 Name.starts_with(
"compress.b") ||
189 Name.starts_with(
"compress.d") ||
190 Name.starts_with(
"compress.p") ||
191 Name.starts_with(
"compress.q") ||
192 Name.starts_with(
"compress.store.") ||
193 Name.starts_with(
"compress.w") ||
194 Name.starts_with(
"conflict.") ||
195 Name.starts_with(
"cvtdq2pd.") ||
196 Name.starts_with(
"cvtdq2ps.") ||
197 Name ==
"cvtpd2dq.256" ||
198 Name ==
"cvtpd2ps.256" ||
199 Name ==
"cvtps2pd.128" ||
200 Name ==
"cvtps2pd.256" ||
201 Name.starts_with(
"cvtqq2pd.") ||
202 Name ==
"cvtqq2ps.256" ||
203 Name ==
"cvtqq2ps.512" ||
204 Name ==
"cvttpd2dq.256" ||
205 Name ==
"cvttps2dq.128" ||
206 Name ==
"cvttps2dq.256" ||
207 Name.starts_with(
"cvtudq2pd.") ||
208 Name.starts_with(
"cvtudq2ps.") ||
209 Name.starts_with(
"cvtuqq2pd.") ||
210 Name ==
"cvtuqq2ps.256" ||
211 Name ==
"cvtuqq2ps.512" ||
212 Name.starts_with(
"dbpsadbw.") ||
213 Name.starts_with(
"div.p") ||
214 Name.starts_with(
"expand.b") ||
215 Name.starts_with(
"expand.d") ||
216 Name.starts_with(
"expand.load.") ||
217 Name.starts_with(
"expand.p") ||
218 Name.starts_with(
"expand.q") ||
219 Name.starts_with(
"expand.w") ||
220 Name.starts_with(
"fpclass.p") ||
221 Name.starts_with(
"insert") ||
222 Name.starts_with(
"load.") ||
223 Name.starts_with(
"loadu.") ||
224 Name.starts_with(
"lzcnt.") ||
225 Name.starts_with(
"max.p") ||
226 Name.starts_with(
"min.p") ||
227 Name.starts_with(
"movddup") ||
228 Name.starts_with(
"move.s") ||
229 Name.starts_with(
"movshdup") ||
230 Name.starts_with(
"movsldup") ||
231 Name.starts_with(
"mul.p") ||
232 Name.starts_with(
"or.") ||
233 Name.starts_with(
"pabs.") ||
234 Name.starts_with(
"packssdw.") ||
235 Name.starts_with(
"packsswb.") ||
236 Name.starts_with(
"packusdw.") ||
237 Name.starts_with(
"packuswb.") ||
238 Name.starts_with(
"padd.") ||
239 Name.starts_with(
"padds.") ||
240 Name.starts_with(
"paddus.") ||
241 Name.starts_with(
"palignr.") ||
242 Name.starts_with(
"pand.") ||
243 Name.starts_with(
"pandn.") ||
244 Name.starts_with(
"pavg") ||
245 Name.starts_with(
"pbroadcast") ||
246 Name.starts_with(
"pcmpeq.") ||
247 Name.starts_with(
"pcmpgt.") ||
248 Name.starts_with(
"perm.df.") ||
249 Name.starts_with(
"perm.di.") ||
250 Name.starts_with(
"permvar.") ||
251 Name.starts_with(
"pmaddubs.w.") ||
252 Name.starts_with(
"pmaddw.d.") ||
253 Name.starts_with(
"pmax") ||
254 Name.starts_with(
"pmin") ||
255 Name ==
"pmov.qd.256" ||
256 Name ==
"pmov.qd.512" ||
257 Name ==
"pmov.wb.256" ||
258 Name ==
"pmov.wb.512" ||
259 Name.starts_with(
"pmovsx") ||
260 Name.starts_with(
"pmovzx") ||
261 Name.starts_with(
"pmul.dq.") ||
262 Name.starts_with(
"pmul.hr.sw.") ||
263 Name.starts_with(
"pmulh.w.") ||
264 Name.starts_with(
"pmulhu.w.") ||
265 Name.starts_with(
"pmull.") ||
266 Name.starts_with(
"pmultishift.qb.") ||
267 Name.starts_with(
"pmulu.dq.") ||
268 Name.starts_with(
"por.") ||
269 Name.starts_with(
"prol.") ||
270 Name.starts_with(
"prolv.") ||
271 Name.starts_with(
"pror.") ||
272 Name.starts_with(
"prorv.") ||
273 Name.starts_with(
"pshuf.b.") ||
274 Name.starts_with(
"pshuf.d.") ||
275 Name.starts_with(
"pshufh.w.") ||
276 Name.starts_with(
"pshufl.w.") ||
277 Name.starts_with(
"psll.d") ||
278 Name.starts_with(
"psll.q") ||
279 Name.starts_with(
"psll.w") ||
280 Name.starts_with(
"pslli") ||
281 Name.starts_with(
"psllv") ||
282 Name.starts_with(
"psra.d") ||
283 Name.starts_with(
"psra.q") ||
284 Name.starts_with(
"psra.w") ||
285 Name.starts_with(
"psrai") ||
286 Name.starts_with(
"psrav") ||
287 Name.starts_with(
"psrl.d") ||
288 Name.starts_with(
"psrl.q") ||
289 Name.starts_with(
"psrl.w") ||
290 Name.starts_with(
"psrli") ||
291 Name.starts_with(
"psrlv") ||
292 Name.starts_with(
"psub.") ||
293 Name.starts_with(
"psubs.") ||
294 Name.starts_with(
"psubus.") ||
295 Name.starts_with(
"pternlog.") ||
296 Name.starts_with(
"punpckh") ||
297 Name.starts_with(
"punpckl") ||
298 Name.starts_with(
"pxor.") ||
299 Name.starts_with(
"shuf.f") ||
300 Name.starts_with(
"shuf.i") ||
301 Name.starts_with(
"shuf.p") ||
302 Name.starts_with(
"sqrt.p") ||
303 Name.starts_with(
"store.b.") ||
304 Name.starts_with(
"store.d.") ||
305 Name.starts_with(
"store.p") ||
306 Name.starts_with(
"store.q.") ||
307 Name.starts_with(
"store.w.") ||
308 Name ==
"store.ss" ||
309 Name.starts_with(
"storeu.") ||
310 Name.starts_with(
"sub.p") ||
311 Name.starts_with(
"ucmp.") ||
312 Name.starts_with(
"unpckh.") ||
313 Name.starts_with(
"unpckl.") ||
314 Name.starts_with(
"valign.") ||
315 Name ==
"vcvtph2ps.128" ||
316 Name ==
"vcvtph2ps.256" ||
317 Name.starts_with(
"vextract") ||
318 Name.starts_with(
"vfmadd.") ||
319 Name.starts_with(
"vfmaddsub.") ||
320 Name.starts_with(
"vfnmadd.") ||
321 Name.starts_with(
"vfnmsub.") ||
322 Name.starts_with(
"vpdpbusd.") ||
323 Name.starts_with(
"vpdpbusds.") ||
324 Name.starts_with(
"vpdpwssd.") ||
325 Name.starts_with(
"vpdpwssds.") ||
326 Name.starts_with(
"vpermi2var.") ||
327 Name.starts_with(
"vpermil.p") ||
328 Name.starts_with(
"vpermilvar.") ||
329 Name.starts_with(
"vpermt2var.") ||
330 Name.starts_with(
"vpmadd52") ||
331 Name.starts_with(
"vpshld.") ||
332 Name.starts_with(
"vpshldv.") ||
333 Name.starts_with(
"vpshrd.") ||
334 Name.starts_with(
"vpshrdv.") ||
335 Name.starts_with(
"vpshufbitqmb.") ||
336 Name.starts_with(
"xor."));
338 if (
Name.consume_front(
"mask3."))
340 return (
Name.starts_with(
"vfmadd.") ||
341 Name.starts_with(
"vfmaddsub.") ||
342 Name.starts_with(
"vfmsub.") ||
343 Name.starts_with(
"vfmsubadd.") ||
344 Name.starts_with(
"vfnmsub."));
346 if (
Name.consume_front(
"maskz."))
348 return (
Name.starts_with(
"pternlog.") ||
349 Name.starts_with(
"vfmadd.") ||
350 Name.starts_with(
"vfmaddsub.") ||
351 Name.starts_with(
"vpdpbusd.") ||
352 Name.starts_with(
"vpdpbusds.") ||
353 Name.starts_with(
"vpdpwssd.") ||
354 Name.starts_with(
"vpdpwssds.") ||
355 Name.starts_with(
"vpermt2var.") ||
356 Name.starts_with(
"vpmadd52") ||
357 Name.starts_with(
"vpshldv.") ||
358 Name.starts_with(
"vpshrdv."));
361 return (
Name ==
"movntdqa" ||
362 Name ==
"pmul.dq.512" ||
363 Name ==
"pmulu.dq.512" ||
364 Name.starts_with(
"broadcastm") ||
365 Name.starts_with(
"cmp.p") ||
366 Name.starts_with(
"cvtb2mask.") ||
367 Name.starts_with(
"cvtd2mask.") ||
368 Name.starts_with(
"cvtmask2") ||
369 Name.starts_with(
"cvtq2mask.") ||
370 Name ==
"cvtusi2sd" ||
371 Name.starts_with(
"cvtw2mask.") ||
376 Name ==
"kortestc.w" ||
377 Name ==
"kortestz.w" ||
378 Name.starts_with(
"kunpck") ||
381 Name.starts_with(
"padds.") ||
382 Name.starts_with(
"pbroadcast") ||
383 Name.starts_with(
"prol") ||
384 Name.starts_with(
"pror") ||
385 Name.starts_with(
"psll.dq") ||
386 Name.starts_with(
"psrl.dq") ||
387 Name.starts_with(
"psubs.") ||
388 Name.starts_with(
"ptestm") ||
389 Name.starts_with(
"ptestnm") ||
390 Name.starts_with(
"storent.") ||
391 Name.starts_with(
"vbroadcast.s") ||
392 Name.starts_with(
"vpshld.") ||
393 Name.starts_with(
"vpshrd."));
396 if (
Name.consume_front(
"fma."))
397 return (
Name.starts_with(
"vfmadd.") ||
398 Name.starts_with(
"vfmsub.") ||
399 Name.starts_with(
"vfmsubadd.") ||
400 Name.starts_with(
"vfnmadd.") ||
401 Name.starts_with(
"vfnmsub."));
403 if (
Name.consume_front(
"fma4."))
404 return Name.starts_with(
"vfmadd.s");
406 if (
Name.consume_front(
"sse."))
407 return (
Name ==
"add.ss" ||
408 Name ==
"cvtsi2ss" ||
409 Name ==
"cvtsi642ss" ||
412 Name.starts_with(
"sqrt.p") ||
414 Name.starts_with(
"storeu.") ||
417 if (
Name.consume_front(
"sse2."))
418 return (
Name ==
"add.sd" ||
419 Name ==
"cvtdq2pd" ||
420 Name ==
"cvtdq2ps" ||
421 Name ==
"cvtps2pd" ||
422 Name ==
"cvtsi2sd" ||
423 Name ==
"cvtsi642sd" ||
424 Name ==
"cvtss2sd" ||
427 Name.starts_with(
"padds.") ||
428 Name.starts_with(
"paddus.") ||
429 Name.starts_with(
"pcmpeq.") ||
430 Name.starts_with(
"pcmpgt.") ||
435 Name ==
"pmulu.dq" ||
436 Name.starts_with(
"pshuf") ||
437 Name.starts_with(
"psll.dq") ||
438 Name.starts_with(
"psrl.dq") ||
439 Name.starts_with(
"psubs.") ||
440 Name.starts_with(
"psubus.") ||
441 Name.starts_with(
"sqrt.p") ||
443 Name ==
"storel.dq" ||
444 Name.starts_with(
"storeu.") ||
447 if (
Name.consume_front(
"sse41."))
448 return (
Name.starts_with(
"blendp") ||
449 Name ==
"movntdqa" ||
459 Name.starts_with(
"pmovsx") ||
460 Name.starts_with(
"pmovzx") ||
463 if (
Name.consume_front(
"sse42."))
464 return Name ==
"crc32.64.8";
466 if (
Name.consume_front(
"sse4a."))
467 return Name.starts_with(
"movnt.");
469 if (
Name.consume_front(
"ssse3."))
470 return (
Name ==
"pabs.b.128" ||
471 Name ==
"pabs.d.128" ||
472 Name ==
"pabs.w.128");
474 if (
Name.consume_front(
"xop."))
475 return (
Name ==
"vpcmov" ||
476 Name ==
"vpcmov.256" ||
477 Name.starts_with(
"vpcom") ||
478 Name.starts_with(
"vprot"));
480 return (
Name ==
"addcarry.u32" ||
481 Name ==
"addcarry.u64" ||
482 Name ==
"addcarryx.u32" ||
483 Name ==
"addcarryx.u64" ||
484 Name ==
"subborrow.u32" ||
485 Name ==
"subborrow.u64" ||
486 Name.starts_with(
"vcvtph2ps."));
492 if (!
Name.consume_front(
"x86."))
500 if (
Name ==
"rdtscp") {
502 if (
F->getFunctionType()->getNumParams() == 0)
507 Intrinsic::x86_rdtscp);
514 if (
Name.consume_front(
"sse41.ptest")) {
516 .
Case(
"c", Intrinsic::x86_sse41_ptestc)
517 .
Case(
"z", Intrinsic::x86_sse41_ptestz)
518 .
Case(
"nzc", Intrinsic::x86_sse41_ptestnzc)
531 .
Case(
"sse41.insertps", Intrinsic::x86_sse41_insertps)
532 .
Case(
"sse41.dppd", Intrinsic::x86_sse41_dppd)
533 .
Case(
"sse41.dpps", Intrinsic::x86_sse41_dpps)
534 .
Case(
"sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
535 .
Case(
"avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
536 .
Case(
"avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
541 if (
Name.consume_front(
"avx512.mask.cmp.")) {
544 .
Case(
"pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
545 .
Case(
"pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
546 .
Case(
"pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
547 .
Case(
"ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
548 .
Case(
"ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
549 .
Case(
"ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
556 if (
Name.consume_front(
"avx512bf16.")) {
559 .
Case(
"cvtne2ps2bf16.128",
560 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
561 .
Case(
"cvtne2ps2bf16.256",
562 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
563 .
Case(
"cvtne2ps2bf16.512",
564 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
565 .
Case(
"mask.cvtneps2bf16.128",
566 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
567 .
Case(
"cvtneps2bf16.256",
568 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
569 .
Case(
"cvtneps2bf16.512",
570 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
577 .
Case(
"dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
578 .
Case(
"dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
579 .
Case(
"dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
586 if (
Name.consume_front(
"xop.")) {
588 if (
Name.starts_with(
"vpermil2")) {
591 auto Idx =
F->getFunctionType()->getParamType(2);
592 if (
Idx->isFPOrFPVectorTy()) {
593 unsigned IdxSize =
Idx->getPrimitiveSizeInBits();
594 unsigned EltSize =
Idx->getScalarSizeInBits();
595 if (EltSize == 64 && IdxSize == 128)
596 ID = Intrinsic::x86_xop_vpermil2pd;
597 else if (EltSize == 32 && IdxSize == 128)
598 ID = Intrinsic::x86_xop_vpermil2ps;
599 else if (EltSize == 64 && IdxSize == 256)
600 ID = Intrinsic::x86_xop_vpermil2pd_256;
602 ID = Intrinsic::x86_xop_vpermil2ps_256;
604 }
else if (
F->arg_size() == 2)
607 .
Case(
"vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
608 .
Case(
"vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
619 if (
Name ==
"seh.recoverfp") {
621 Intrinsic::eh_recoverfp);
633 if (
Name.starts_with(
"rbit")) {
636 F->getParent(), Intrinsic::bitreverse,
F->arg_begin()->getType());
640 if (
Name ==
"thread.pointer") {
643 Intrinsic::thread_pointer);
647 bool Neon =
Name.consume_front(
"neon.");
652 if (
Name.consume_front(
"bfdot.")) {
656 .
Cases(
"v2f32.v8i8",
"v4f32.v16i8",
661 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
662 assert((OperandWidth == 64 || OperandWidth == 128) &&
663 "Unexpected operand width");
665 std::array<Type *, 2> Tys{
676 if (
Name.consume_front(
"bfm")) {
678 if (
Name.consume_back(
".v4f32.v16i8")) {
718 F->arg_begin()->getType());
722 if (
Name.consume_front(
"vst")) {
724 static const Regex vstRegex(
"^([1234]|[234]lane)\\.v[a-z0-9]*$");
728 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
729 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
732 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
733 Intrinsic::arm_neon_vst4lane};
735 auto fArgs =
F->getFunctionType()->params();
736 Type *Tys[] = {fArgs[0], fArgs[1]};
739 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
742 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
751 if (
Name.consume_front(
"mve.")) {
753 if (
Name ==
"vctp64") {
754 if (cast<FixedVectorType>(
F->getReturnType())->getNumElements() == 4) {
764 if (
Name.consume_back(
".v4i1")) {
766 if (
Name.consume_back(
".predicated.v2i64.v4i32"))
768 return Name ==
"mull.int" ||
Name ==
"vqdmull";
770 if (
Name.consume_back(
".v2i64")) {
772 bool IsGather =
Name.consume_front(
"vldr.gather.");
773 if (IsGather ||
Name.consume_front(
"vstr.scatter.")) {
774 if (
Name.consume_front(
"base.")) {
776 Name.consume_front(
"wb.");
779 return Name ==
"predicated.v2i64";
782 if (
Name.consume_front(
"offset.predicated."))
783 return Name == (IsGather ?
"v2i64.p0i64" :
"p0i64.v2i64") ||
784 Name == (IsGather ?
"v2i64.p0" :
"p0.v2i64");
797 if (
Name.consume_front(
"cde.vcx")) {
799 if (
Name.consume_back(
".predicated.v2i64.v4i1"))
816 F->arg_begin()->getType());
820 if (
Name.starts_with(
"addp")) {
822 if (
F->arg_size() != 2)
824 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
825 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
827 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
833 if (
Name.consume_front(
"sve.")) {
835 if (
Name.consume_front(
"bf")) {
836 if (
Name.consume_back(
".lane")) {
840 .
Case(
"dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
841 .
Case(
"mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
842 .
Case(
"mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
854 if (
Name ==
"fcvt.bf16f32" ||
Name ==
"fcvtnt.bf16f32") {
859 if (
Name.consume_front(
"addqv")) {
861 if (!
F->getReturnType()->isFPOrFPVectorTy())
864 auto Args =
F->getFunctionType()->params();
865 Type *Tys[] = {
F->getReturnType(), Args[1]};
867 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
871 if (
Name.consume_front(
"ld")) {
873 static const Regex LdRegex(
"^[234](.nxv[a-z0-9]+|$)");
876 cast<VectorType>(
F->getReturnType())->getElementType();
878 cast<VectorType>(
F->arg_begin()->getType())->getElementCount();
879 Type *Ty = VectorType::get(ScalarTy, EC);
881 Intrinsic::aarch64_sve_ld2_sret,
882 Intrinsic::aarch64_sve_ld3_sret,
883 Intrinsic::aarch64_sve_ld4_sret,
886 LoadIDs[
Name[0] -
'2'], Ty);
892 if (
Name.consume_front(
"tuple.")) {
894 if (
Name.starts_with(
"get")) {
896 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
898 F->getParent(), Intrinsic::vector_extract, Tys);
902 if (
Name.starts_with(
"set")) {
904 auto Args =
F->getFunctionType()->params();
905 Type *Tys[] = {Args[0], Args[2], Args[1]};
907 F->getParent(), Intrinsic::vector_insert, Tys);
911 static const Regex CreateTupleRegex(
"^create[234](.nxv[a-z0-9]+|$)");
914 auto Args =
F->getFunctionType()->params();
915 Type *Tys[] = {
F->getReturnType(), Args[1]};
917 F->getParent(), Intrinsic::vector_insert, Tys);
929 if (
Name.consume_front(
"abs."))
931 .
Case(
"bf16", Intrinsic::nvvm_abs_bf16)
932 .
Case(
"bf16x2", Intrinsic::nvvm_abs_bf16x2)
935 if (
Name.consume_front(
"fma.rn."))
937 .
Case(
"bf16", Intrinsic::nvvm_fma_rn_bf16)
938 .
Case(
"bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
939 .
Case(
"ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)
940 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)
941 .
Case(
"ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)
942 .
Case(
"ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)
943 .
Case(
"ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)
944 .
Case(
"ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)
945 .
Case(
"relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
946 .
Case(
"relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
947 .
Case(
"sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)
948 .
Case(
"sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)
951 if (
Name.consume_front(
"fmax."))
953 .
Case(
"bf16", Intrinsic::nvvm_fmax_bf16)
954 .
Case(
"bf16x2", Intrinsic::nvvm_fmax_bf16x2)
955 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
956 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
957 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
958 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
959 .
Case(
"ftz.nan.xorsign.abs.bf16",
960 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
961 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
962 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
963 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
964 .
Case(
"ftz.xorsign.abs.bf16x2",
965 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
966 .
Case(
"nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
967 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
968 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
969 .
Case(
"nan.xorsign.abs.bf16x2",
970 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
971 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
972 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
975 if (
Name.consume_front(
"fmin."))
977 .
Case(
"bf16", Intrinsic::nvvm_fmin_bf16)
978 .
Case(
"bf16x2", Intrinsic::nvvm_fmin_bf16x2)
979 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
980 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
981 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
982 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
983 .
Case(
"ftz.nan.xorsign.abs.bf16",
984 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
985 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
986 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
987 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
988 .
Case(
"ftz.xorsign.abs.bf16x2",
989 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
990 .
Case(
"nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
991 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
992 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
993 .
Case(
"nan.xorsign.abs.bf16x2",
994 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
995 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
996 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
999 if (
Name.consume_front(
"neg."))
1001 .
Case(
"bf16", Intrinsic::nvvm_neg_bf16)
1002 .
Case(
"bf16x2", Intrinsic::nvvm_neg_bf16x2)
1009 bool CanUpgradeDebugIntrinsicsToRecords) {
1010 assert(
F &&
"Illegal to upgrade a non-existent Function.");
1015 if (!
Name.consume_front(
"llvm.") ||
Name.empty())
1021 bool IsArm =
Name.consume_front(
"arm.");
1022 if (IsArm ||
Name.consume_front(
"aarch64.")) {
1028 if (
Name.consume_front(
"amdgcn.")) {
1029 if (
Name ==
"alignbit") {
1032 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1036 if (
Name.consume_front(
"atomic.")) {
1037 if (
Name.starts_with(
"inc") ||
Name.starts_with(
"dec")) {
1046 if (
Name.consume_front(
"ds.") ||
Name.consume_front(
"global.atomic.") ||
1047 Name.consume_front(
"flat.atomic.")) {
1048 if (
Name.starts_with(
"fadd") ||
1050 (
Name.starts_with(
"fmin") && !
Name.starts_with(
"fmin.num")) ||
1051 (
Name.starts_with(
"fmax") && !
Name.starts_with(
"fmax.num"))) {
1059 if (
Name.starts_with(
"ldexp.")) {
1062 F->getParent(), Intrinsic::ldexp,
1063 {F->getReturnType(), F->getArg(1)->getType()});
1072 if (
F->arg_size() == 1) {
1080 F->arg_begin()->getType());
1085 if (
F->arg_size() == 2 &&
Name ==
"coro.end") {
1088 Intrinsic::coro_end);
1095 if (
Name.consume_front(
"dbg.")) {
1097 if (CanUpgradeDebugIntrinsicsToRecords &&
1098 F->getParent()->IsNewDbgInfoFormat) {
1099 if (
Name ==
"addr" ||
Name ==
"value" ||
Name ==
"assign" ||
1100 Name ==
"declare" ||
Name ==
"label") {
1109 if (
Name ==
"addr" || (
Name ==
"value" &&
F->arg_size() == 4)) {
1112 Intrinsic::dbg_value);
1119 if (
Name.consume_front(
"experimental.vector.")) {
1125 .
StartsWith(
"extract.", Intrinsic::vector_extract)
1126 .
StartsWith(
"insert.", Intrinsic::vector_insert)
1127 .
StartsWith(
"splice.", Intrinsic::vector_splice)
1128 .
StartsWith(
"reverse.", Intrinsic::vector_reverse)
1129 .
StartsWith(
"interleave2.", Intrinsic::vector_interleave2)
1130 .
StartsWith(
"deinterleave2.", Intrinsic::vector_deinterleave2)
1133 const auto *FT =
F->getFunctionType();
1135 if (
ID == Intrinsic::vector_extract ||
1136 ID == Intrinsic::vector_interleave2)
1138 Tys.push_back(FT->getReturnType());
1139 if (
ID != Intrinsic::vector_interleave2)
1140 Tys.push_back(FT->getParamType(0));
1141 if (
ID == Intrinsic::vector_insert)
1143 Tys.push_back(FT->getParamType(1));
1149 if (
Name.consume_front(
"reduce.")) {
1151 static const Regex R(
"^([a-z]+)\\.[a-z][0-9]+");
1154 .
Case(
"add", Intrinsic::vector_reduce_add)
1155 .
Case(
"mul", Intrinsic::vector_reduce_mul)
1156 .
Case(
"and", Intrinsic::vector_reduce_and)
1157 .
Case(
"or", Intrinsic::vector_reduce_or)
1158 .
Case(
"xor", Intrinsic::vector_reduce_xor)
1159 .
Case(
"smax", Intrinsic::vector_reduce_smax)
1160 .
Case(
"smin", Intrinsic::vector_reduce_smin)
1161 .
Case(
"umax", Intrinsic::vector_reduce_umax)
1162 .
Case(
"umin", Intrinsic::vector_reduce_umin)
1163 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
1164 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
1169 static const Regex R2(
"^v2\\.([a-z]+)\\.[fi][0-9]+");
1174 .
Case(
"fadd", Intrinsic::vector_reduce_fadd)
1175 .
Case(
"fmul", Intrinsic::vector_reduce_fmul)
1180 auto Args =
F->getFunctionType()->params();
1182 {Args[V2 ? 1 : 0]});
1189 if (
Name.consume_front(
"experimental.stepvector.")) {
1193 F->getParent(),
ID,
F->getFunctionType()->getReturnType());
1198 if (
Name.starts_with(
"flt.rounds")) {
1201 Intrinsic::get_rounding);
1206 if (
Name.starts_with(
"invariant.group.barrier")) {
1208 auto Args =
F->getFunctionType()->params();
1209 Type* ObjectPtr[1] = {Args[0]};
1212 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1221 .StartsWith(
"memcpy.", Intrinsic::memcpy)
1222 .StartsWith(
"memmove.", Intrinsic::memmove)
1224 if (
F->arg_size() == 5) {
1228 F->getFunctionType()->params().slice(0, 3);
1234 if (
Name.starts_with(
"memset.") &&
F->arg_size() == 5) {
1237 const auto *FT =
F->getFunctionType();
1238 Type *ParamTypes[2] = {
1239 FT->getParamType(0),
1243 Intrinsic::memset, ParamTypes);
1249 if (
Name.consume_front(
"nvvm.")) {
1251 if (
F->arg_size() == 1) {
1254 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
1255 .
Case(
"clz.i", Intrinsic::ctlz)
1256 .
Case(
"popc.i", Intrinsic::ctpop)
1260 {F->getReturnType()});
1266 if (!
F->getReturnType()->getScalarType()->isBFloatTy()) {
1278 bool Expand =
false;
1279 if (
Name.consume_front(
"abs."))
1281 Expand =
Name ==
"i" ||
Name ==
"ll";
1282 else if (
Name ==
"clz.ll" ||
Name ==
"popc.ll" ||
Name ==
"h2f")
1284 else if (
Name.consume_front(
"max.") ||
Name.consume_front(
"min."))
1288 else if (
Name.consume_front(
"atomic.load.add."))
1290 Expand =
Name.starts_with(
"f32.p") ||
Name.starts_with(
"f64.p");
1291 else if (
Name.consume_front(
"bitcast."))
1295 else if (
Name.consume_front(
"rotate."))
1297 Expand =
Name ==
"b32" ||
Name ==
"b64" ||
Name ==
"right.b64";
1298 else if (
Name.consume_front(
"ptr.gen.to."))
1300 Expand =
Name.starts_with(
"local") ||
Name.starts_with(
"shared") ||
1301 Name.starts_with(
"global") ||
Name.starts_with(
"constant");
1302 else if (
Name.consume_front(
"ptr."))
1305 (
Name.consume_front(
"local") ||
Name.consume_front(
"shared") ||
1306 Name.consume_front(
"global") ||
Name.consume_front(
"constant")) &&
1307 Name.starts_with(
".to.gen");
1308 else if (
Name.consume_front(
"ldg.global."))
1310 Expand = (
Name.starts_with(
"i.") ||
Name.starts_with(
"f.") ||
1311 Name.starts_with(
"p."));
1326 if (
Name.starts_with(
"objectsize.")) {
1327 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
1328 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
1333 Intrinsic::objectsize, Tys);
1340 if (
Name.starts_with(
"ptr.annotation.") &&
F->arg_size() == 4) {
1343 F->getParent(), Intrinsic::ptr_annotation,
1344 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1350 if (
Name.consume_front(
"riscv.")) {
1353 .
Case(
"aes32dsi", Intrinsic::riscv_aes32dsi)
1354 .
Case(
"aes32dsmi", Intrinsic::riscv_aes32dsmi)
1355 .
Case(
"aes32esi", Intrinsic::riscv_aes32esi)
1356 .
Case(
"aes32esmi", Intrinsic::riscv_aes32esmi)
1359 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1372 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1373 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1382 .
StartsWith(
"sha256sig0", Intrinsic::riscv_sha256sig0)
1383 .
StartsWith(
"sha256sig1", Intrinsic::riscv_sha256sig1)
1384 .
StartsWith(
"sha256sum0", Intrinsic::riscv_sha256sum0)
1385 .
StartsWith(
"sha256sum1", Intrinsic::riscv_sha256sum1)
1390 if (
F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1402 if (
Name ==
"stackprotectorcheck") {
1409 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
1412 F->getParent(), Intrinsic::var_annotation,
1413 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1420 if (
Name.consume_front(
"wasm.")) {
1423 .
StartsWith(
"fma.", Intrinsic::wasm_relaxed_madd)
1424 .
StartsWith(
"fms.", Intrinsic::wasm_relaxed_nmadd)
1425 .
StartsWith(
"laneselect.", Intrinsic::wasm_relaxed_laneselect)
1430 F->getReturnType());
1434 if (
Name.consume_front(
"dot.i8x16.i7x16.")) {
1436 .
Case(
"signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1438 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1456 auto *
ST = dyn_cast<StructType>(
F->getReturnType());
1457 if (ST && (!
ST->isLiteral() ||
ST->isPacked()) &&
1466 auto *FT =
F->getFunctionType();
1468 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1469 std::string
Name =
F->getName().str();
1472 Name,
F->getParent());
1483 if (Result != std::nullopt) {
1496 bool CanUpgradeDebugIntrinsicsToRecords) {
1500 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1512 GV->
getName() ==
"llvm.global_dtors")) ||
1527 unsigned N =
Init->getNumOperands();
1528 std::vector<Constant *> NewCtors(
N);
1529 for (
unsigned i = 0; i !=
N; ++i) {
1530 auto Ctor = cast<Constant>(
Init->getOperand(i));
1532 Ctor->getAggregateElement(1),
1545 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1546 unsigned NumElts = ResultTy->getNumElements() * 8;
1560 for (
unsigned l = 0; l != NumElts; l += 16)
1561 for (
unsigned i = 0; i != 16; ++i) {
1562 unsigned Idx = NumElts + i - Shift;
1564 Idx -= NumElts - 16;
1565 Idxs[l + i] =
Idx + l;
1579 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1580 unsigned NumElts = ResultTy->getNumElements() * 8;
1594 for (
unsigned l = 0; l != NumElts; l += 16)
1595 for (
unsigned i = 0; i != 16; ++i) {
1596 unsigned Idx = i + Shift;
1598 Idx += NumElts - 16;
1599 Idxs[l + i] =
Idx + l;
1620 for (
unsigned i = 0; i != NumElts; ++i)
1632 if (
const auto *
C = dyn_cast<Constant>(Mask))
1633 if (
C->isAllOnesValue())
1637 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1644 if (
const auto *
C = dyn_cast<Constant>(Mask))
1645 if (
C->isAllOnesValue())
1649 Mask->getType()->getIntegerBitWidth());
1662 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1664 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1665 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1666 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1671 ShiftVal &= (NumElts - 1);
1680 if (ShiftVal > 16) {
1688 for (
unsigned l = 0; l < NumElts; l += 16) {
1689 for (
unsigned i = 0; i != 16; ++i) {
1690 unsigned Idx = ShiftVal + i;
1691 if (!IsVALIGN &&
Idx >= 16)
1692 Idx += NumElts - 16;
1693 Indices[l + i] =
Idx + l;
1698 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
1704 bool ZeroMask,
bool IndexForm) {
1710 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1711 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1712 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1713 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1714 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1715 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1716 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1717 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1718 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1719 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1720 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1721 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1722 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1723 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1724 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1725 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1726 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1727 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1728 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1729 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1730 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1731 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1732 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1733 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1734 else if (VecWidth == 128 && EltWidth == 16)
1735 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1736 else if (VecWidth == 256 && EltWidth == 16)
1737 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1738 else if (VecWidth == 512 && EltWidth == 16)
1739 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1740 else if (VecWidth == 128 && EltWidth == 8)
1741 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1742 else if (VecWidth == 256 && EltWidth == 8)
1743 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1744 else if (VecWidth == 512 && EltWidth == 8)
1745 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1779 bool IsRotateRight) {
1788 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1793 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1813 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1816 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1819 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1822 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1825 Pred = ICmpInst::ICMP_EQ;
1828 Pred = ICmpInst::ICMP_NE;
1844 bool IsShiftRight,
bool ZeroMask) {
1857 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1862 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1878 const Align Alignment =
1880 ?
Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1884 if (
const auto *
C = dyn_cast<Constant>(Mask))
1885 if (
C->isAllOnesValue())
1889 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1897 const Align Alignment =
1905 if (
const auto *
C = dyn_cast<Constant>(Mask))
1906 if (
C->isAllOnesValue())
1910 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1919 {Op0, Builder.
getInt1(
false)});
1934 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1941 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1957 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1959 const auto *
C = dyn_cast<Constant>(Mask);
1960 if (!
C || !
C->isAllOnesValue())
1966 for (
unsigned i = 0; i != NumElts; ++i)
1968 for (
unsigned i = NumElts; i != 8; ++i)
1969 Indices[i] = NumElts + i % NumElts;
1980 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1986 }
else if (
CC == 7) {
1993 case 0: Pred = ICmpInst::ICMP_EQ;
break;
1994 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
1995 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
1996 case 4: Pred = ICmpInst::ICMP_NE;
break;
1997 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
1998 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
2033 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
2035 return Builder.
CreateSExt(Mask, ReturnOp,
"vpmovm2");
2046 if (
Name.starts_with(
"max.p")) {
2047 if (VecWidth == 128 && EltWidth == 32)
2048 IID = Intrinsic::x86_sse_max_ps;
2049 else if (VecWidth == 128 && EltWidth == 64)
2050 IID = Intrinsic::x86_sse2_max_pd;
2051 else if (VecWidth == 256 && EltWidth == 32)
2052 IID = Intrinsic::x86_avx_max_ps_256;
2053 else if (VecWidth == 256 && EltWidth == 64)
2054 IID = Intrinsic::x86_avx_max_pd_256;
2057 }
else if (
Name.starts_with(
"min.p")) {
2058 if (VecWidth == 128 && EltWidth == 32)
2059 IID = Intrinsic::x86_sse_min_ps;
2060 else if (VecWidth == 128 && EltWidth == 64)
2061 IID = Intrinsic::x86_sse2_min_pd;
2062 else if (VecWidth == 256 && EltWidth == 32)
2063 IID = Intrinsic::x86_avx_min_ps_256;
2064 else if (VecWidth == 256 && EltWidth == 64)
2065 IID = Intrinsic::x86_avx_min_pd_256;
2068 }
else if (
Name.starts_with(
"pshuf.b.")) {
2069 if (VecWidth == 128)
2070 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2071 else if (VecWidth == 256)
2072 IID = Intrinsic::x86_avx2_pshuf_b;
2073 else if (VecWidth == 512)
2074 IID = Intrinsic::x86_avx512_pshuf_b_512;
2077 }
else if (
Name.starts_with(
"pmul.hr.sw.")) {
2078 if (VecWidth == 128)
2079 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2080 else if (VecWidth == 256)
2081 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2082 else if (VecWidth == 512)
2083 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2086 }
else if (
Name.starts_with(
"pmulh.w.")) {
2087 if (VecWidth == 128)
2088 IID = Intrinsic::x86_sse2_pmulh_w;
2089 else if (VecWidth == 256)
2090 IID = Intrinsic::x86_avx2_pmulh_w;
2091 else if (VecWidth == 512)
2092 IID = Intrinsic::x86_avx512_pmulh_w_512;
2095 }
else if (
Name.starts_with(
"pmulhu.w.")) {
2096 if (VecWidth == 128)
2097 IID = Intrinsic::x86_sse2_pmulhu_w;
2098 else if (VecWidth == 256)
2099 IID = Intrinsic::x86_avx2_pmulhu_w;
2100 else if (VecWidth == 512)
2101 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2104 }
else if (
Name.starts_with(
"pmaddw.d.")) {
2105 if (VecWidth == 128)
2106 IID = Intrinsic::x86_sse2_pmadd_wd;
2107 else if (VecWidth == 256)
2108 IID = Intrinsic::x86_avx2_pmadd_wd;
2109 else if (VecWidth == 512)
2110 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2113 }
else if (
Name.starts_with(
"pmaddubs.w.")) {
2114 if (VecWidth == 128)
2115 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2116 else if (VecWidth == 256)
2117 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2118 else if (VecWidth == 512)
2119 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2122 }
else if (
Name.starts_with(
"packsswb.")) {
2123 if (VecWidth == 128)
2124 IID = Intrinsic::x86_sse2_packsswb_128;
2125 else if (VecWidth == 256)
2126 IID = Intrinsic::x86_avx2_packsswb;
2127 else if (VecWidth == 512)
2128 IID = Intrinsic::x86_avx512_packsswb_512;
2131 }
else if (
Name.starts_with(
"packssdw.")) {
2132 if (VecWidth == 128)
2133 IID = Intrinsic::x86_sse2_packssdw_128;
2134 else if (VecWidth == 256)
2135 IID = Intrinsic::x86_avx2_packssdw;
2136 else if (VecWidth == 512)
2137 IID = Intrinsic::x86_avx512_packssdw_512;
2140 }
else if (
Name.starts_with(
"packuswb.")) {
2141 if (VecWidth == 128)
2142 IID = Intrinsic::x86_sse2_packuswb_128;
2143 else if (VecWidth == 256)
2144 IID = Intrinsic::x86_avx2_packuswb;
2145 else if (VecWidth == 512)
2146 IID = Intrinsic::x86_avx512_packuswb_512;
2149 }
else if (
Name.starts_with(
"packusdw.")) {
2150 if (VecWidth == 128)
2151 IID = Intrinsic::x86_sse41_packusdw;
2152 else if (VecWidth == 256)
2153 IID = Intrinsic::x86_avx2_packusdw;
2154 else if (VecWidth == 512)
2155 IID = Intrinsic::x86_avx512_packusdw_512;
2158 }
else if (
Name.starts_with(
"vpermilvar.")) {
2159 if (VecWidth == 128 && EltWidth == 32)
2160 IID = Intrinsic::x86_avx_vpermilvar_ps;
2161 else if (VecWidth == 128 && EltWidth == 64)
2162 IID = Intrinsic::x86_avx_vpermilvar_pd;
2163 else if (VecWidth == 256 && EltWidth == 32)
2164 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2165 else if (VecWidth == 256 && EltWidth == 64)
2166 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2167 else if (VecWidth == 512 && EltWidth == 32)
2168 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2169 else if (VecWidth == 512 && EltWidth == 64)
2170 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2173 }
else if (
Name ==
"cvtpd2dq.256") {
2174 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2175 }
else if (
Name ==
"cvtpd2ps.256") {
2176 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2177 }
else if (
Name ==
"cvttpd2dq.256") {
2178 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2179 }
else if (
Name ==
"cvttps2dq.128") {
2180 IID = Intrinsic::x86_sse2_cvttps2dq;
2181 }
else if (
Name ==
"cvttps2dq.256") {
2182 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2183 }
else if (
Name.starts_with(
"permvar.")) {
2185 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2186 IID = Intrinsic::x86_avx2_permps;
2187 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2188 IID = Intrinsic::x86_avx2_permd;
2189 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2190 IID = Intrinsic::x86_avx512_permvar_df_256;
2191 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2192 IID = Intrinsic::x86_avx512_permvar_di_256;
2193 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2194 IID = Intrinsic::x86_avx512_permvar_sf_512;
2195 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2196 IID = Intrinsic::x86_avx512_permvar_si_512;
2197 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2198 IID = Intrinsic::x86_avx512_permvar_df_512;
2199 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2200 IID = Intrinsic::x86_avx512_permvar_di_512;
2201 else if (VecWidth == 128 && EltWidth == 16)
2202 IID = Intrinsic::x86_avx512_permvar_hi_128;
2203 else if (VecWidth == 256 && EltWidth == 16)
2204 IID = Intrinsic::x86_avx512_permvar_hi_256;
2205 else if (VecWidth == 512 && EltWidth == 16)
2206 IID = Intrinsic::x86_avx512_permvar_hi_512;
2207 else if (VecWidth == 128 && EltWidth == 8)
2208 IID = Intrinsic::x86_avx512_permvar_qi_128;
2209 else if (VecWidth == 256 && EltWidth == 8)
2210 IID = Intrinsic::x86_avx512_permvar_qi_256;
2211 else if (VecWidth == 512 && EltWidth == 8)
2212 IID = Intrinsic::x86_avx512_permvar_qi_512;
2215 }
else if (
Name.starts_with(
"dbpsadbw.")) {
2216 if (VecWidth == 128)
2217 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2218 else if (VecWidth == 256)
2219 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2220 else if (VecWidth == 512)
2221 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2224 }
else if (
Name.starts_with(
"pmultishift.qb.")) {
2225 if (VecWidth == 128)
2226 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2227 else if (VecWidth == 256)
2228 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2229 else if (VecWidth == 512)
2230 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2233 }
else if (
Name.starts_with(
"conflict.")) {
2234 if (
Name[9] ==
'd' && VecWidth == 128)
2235 IID = Intrinsic::x86_avx512_conflict_d_128;
2236 else if (
Name[9] ==
'd' && VecWidth == 256)
2237 IID = Intrinsic::x86_avx512_conflict_d_256;
2238 else if (
Name[9] ==
'd' && VecWidth == 512)
2239 IID = Intrinsic::x86_avx512_conflict_d_512;
2240 else if (
Name[9] ==
'q' && VecWidth == 128)
2241 IID = Intrinsic::x86_avx512_conflict_q_128;
2242 else if (
Name[9] ==
'q' && VecWidth == 256)
2243 IID = Intrinsic::x86_avx512_conflict_q_256;
2244 else if (
Name[9] ==
'q' && VecWidth == 512)
2245 IID = Intrinsic::x86_avx512_conflict_q_512;
2248 }
else if (
Name.starts_with(
"pavg.")) {
2249 if (
Name[5] ==
'b' && VecWidth == 128)
2250 IID = Intrinsic::x86_sse2_pavg_b;
2251 else if (
Name[5] ==
'b' && VecWidth == 256)
2252 IID = Intrinsic::x86_avx2_pavg_b;
2253 else if (
Name[5] ==
'b' && VecWidth == 512)
2254 IID = Intrinsic::x86_avx512_pavg_b_512;
2255 else if (
Name[5] ==
'w' && VecWidth == 128)
2256 IID = Intrinsic::x86_sse2_pavg_w;
2257 else if (
Name[5] ==
'w' && VecWidth == 256)
2258 IID = Intrinsic::x86_avx2_pavg_w;
2259 else if (
Name[5] ==
'w' && VecWidth == 512)
2260 IID = Intrinsic::x86_avx512_pavg_w_512;
2280 if (AsmStr->find(
"mov\tfp") == 0 &&
2281 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
2282 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
2283 AsmStr->replace(Pos, 1,
";");
2289 Value *Rep =
nullptr;
2291 if (
Name ==
"abs.i" ||
Name ==
"abs.ll") {
2297 }
else if (
Name.starts_with(
"atomic.load.add.f32.p") ||
2298 Name.starts_with(
"atomic.load.add.f64.p")) {
2302 AtomicOrdering::SequentiallyConsistent);
2303 }
else if (
Name.consume_front(
"max.") &&
2312 }
else if (
Name.consume_front(
"min.") &&
2321 }
else if (
Name ==
"clz.ll") {
2328 }
else if (
Name ==
"popc.ll") {
2333 Arg,
nullptr,
"ctpop");
2335 }
else if (
Name ==
"h2f") {
2339 }
else if (
Name.consume_front(
"bitcast.") &&
2343 }
else if (
Name ==
"rotate.b32") {
2347 {Arg, Arg, ShiftAmt});
2348 }
else if (
Name ==
"rotate.b64") {
2353 {Arg, Arg, ZExtShiftAmt});
2354 }
else if (
Name ==
"rotate.right.b64") {
2359 {Arg, Arg, ZExtShiftAmt});
2360 }
else if ((
Name.consume_front(
"ptr.gen.to.") &&
2361 (
Name.starts_with(
"local") ||
Name.starts_with(
"shared") ||
2362 Name.starts_with(
"global") ||
Name.starts_with(
"constant"))) ||
2363 (
Name.consume_front(
"ptr.") &&
2364 (
Name.consume_front(
"local") ||
Name.consume_front(
"shared") ||
2365 Name.consume_front(
"global") ||
2366 Name.consume_front(
"constant")) &&
2367 Name.starts_with(
".to.gen"))) {
2369 }
else if (
Name.consume_front(
"ldg.global")) {
2376 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
2381 !
F->getReturnType()->getScalarType()->isBFloatTy()) {
2395 if (
F->getReturnType()->isIntegerTy())
2406 Value *Rep =
nullptr;
2408 if (
Name.starts_with(
"sse4a.movnt.")) {
2423 SI->setMetadata(LLVMContext::MD_nontemporal,
Node);
2424 }
else if (
Name.starts_with(
"avx.movnt.") ||
2425 Name.starts_with(
"avx512.storent.")) {
2437 SI->setMetadata(LLVMContext::MD_nontemporal,
Node);
2438 }
else if (
Name ==
"sse2.storel.dq") {
2446 }
else if (
Name.starts_with(
"sse.storeu.") ||
2447 Name.starts_with(
"sse2.storeu.") ||
2448 Name.starts_with(
"avx.storeu.")) {
2452 }
else if (
Name ==
"avx512.mask.store.ss") {
2456 }
else if (
Name.starts_with(
"avx512.mask.store")) {
2461 }
else if (
Name.starts_with(
"sse2.pcmp") ||
Name.starts_with(
"avx2.pcmp")) {
2464 bool CmpEq =
Name[9] ==
'e';
2465 Rep = Builder.
CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2468 }
else if (
Name.starts_with(
"avx512.broadcastm")) {
2476 }
else if (
Name ==
"sse.sqrt.ss" ||
Name ==
"sse2.sqrt.sd") {
2481 }
else if (
Name.starts_with(
"avx.sqrt.p") ||
2482 Name.starts_with(
"sse2.sqrt.p") ||
2483 Name.starts_with(
"sse.sqrt.p")) {
2485 {CI->getArgOperand(0)});
2486 }
else if (
Name.starts_with(
"avx512.mask.sqrt.p")) {
2489 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2491 : Intrinsic::x86_avx512_sqrt_pd_512;
2497 {CI->getArgOperand(0)});
2501 }
else if (
Name.starts_with(
"avx512.ptestm") ||
2502 Name.starts_with(
"avx512.ptestnm")) {
2511 : ICmpInst::ICMP_EQ;
2514 }
else if (
Name.starts_with(
"avx512.mask.pbroadcast")) {
2520 }
else if (
Name.starts_with(
"avx512.kunpck")) {
2525 for (
unsigned i = 0; i != NumElts; ++i)
2536 }
else if (
Name ==
"avx512.kand.w") {
2541 }
else if (
Name ==
"avx512.kandn.w") {
2547 }
else if (
Name ==
"avx512.kor.w") {
2552 }
else if (
Name ==
"avx512.kxor.w") {
2557 }
else if (
Name ==
"avx512.kxnor.w") {
2563 }
else if (
Name ==
"avx512.knot.w") {
2567 }
else if (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w") {
2573 if (
Name[14] ==
'c')
2574 C = ConstantInt::getAllOnesValue(Builder.
getInt16Ty());
2576 C = ConstantInt::getNullValue(Builder.
getInt16Ty());
2579 }
else if (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2580 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2581 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2582 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd") {
2585 ConstantInt::get(I32Ty, 0));
2587 ConstantInt::get(I32Ty, 0));
2589 if (
Name.contains(
".add."))
2591 else if (
Name.contains(
".sub."))
2593 else if (
Name.contains(
".mul."))
2598 ConstantInt::get(I32Ty, 0));
2599 }
else if (
Name.starts_with(
"avx512.mask.pcmp")) {
2601 bool CmpEq =
Name[16] ==
'e';
2603 }
else if (
Name.starts_with(
"avx512.mask.vpshufbitqmb.")) {
2611 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
2614 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
2617 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
2624 }
else if (
Name.starts_with(
"avx512.mask.fpclass.p")) {
2629 if (VecWidth == 128 && EltWidth == 32)
2630 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2631 else if (VecWidth == 256 && EltWidth == 32)
2632 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2633 else if (VecWidth == 512 && EltWidth == 32)
2634 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2635 else if (VecWidth == 128 && EltWidth == 64)
2636 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2637 else if (VecWidth == 256 && EltWidth == 64)
2638 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2639 else if (VecWidth == 512 && EltWidth == 64)
2640 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2647 }
else if (
Name.starts_with(
"avx512.cmp.p")) {
2649 Type *OpTy = Args[0]->getType();
2653 if (VecWidth == 128 && EltWidth == 32)
2654 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2655 else if (VecWidth == 256 && EltWidth == 32)
2656 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2657 else if (VecWidth == 512 && EltWidth == 32)
2658 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2659 else if (VecWidth == 128 && EltWidth == 64)
2660 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2661 else if (VecWidth == 256 && EltWidth == 64)
2662 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2663 else if (VecWidth == 512 && EltWidth == 64)
2664 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2669 if (VecWidth == 512)
2671 Args.push_back(Mask);
2674 }
else if (
Name.starts_with(
"avx512.mask.cmp.")) {
2676 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2678 }
else if (
Name.starts_with(
"avx512.mask.ucmp.")) {
2679 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2681 }
else if (
Name.starts_with(
"avx512.cvtb2mask.") ||
2682 Name.starts_with(
"avx512.cvtw2mask.") ||
2683 Name.starts_with(
"avx512.cvtd2mask.") ||
2684 Name.starts_with(
"avx512.cvtq2mask.")) {
2687 Rep = Builder.
CreateICmp(ICmpInst::ICMP_SLT,
Op, Zero);
2689 }
else if (
Name ==
"ssse3.pabs.b.128" ||
Name ==
"ssse3.pabs.w.128" ||
2690 Name ==
"ssse3.pabs.d.128" ||
Name.starts_with(
"avx2.pabs") ||
2691 Name.starts_with(
"avx512.mask.pabs")) {
2693 }
else if (
Name ==
"sse41.pmaxsb" ||
Name ==
"sse2.pmaxs.w" ||
2694 Name ==
"sse41.pmaxsd" ||
Name.starts_with(
"avx2.pmaxs") ||
2695 Name.starts_with(
"avx512.mask.pmaxs")) {
2697 }
else if (
Name ==
"sse2.pmaxu.b" ||
Name ==
"sse41.pmaxuw" ||
2698 Name ==
"sse41.pmaxud" ||
Name.starts_with(
"avx2.pmaxu") ||
2699 Name.starts_with(
"avx512.mask.pmaxu")) {
2701 }
else if (
Name ==
"sse41.pminsb" ||
Name ==
"sse2.pmins.w" ||
2702 Name ==
"sse41.pminsd" ||
Name.starts_with(
"avx2.pmins") ||
2703 Name.starts_with(
"avx512.mask.pmins")) {
2705 }
else if (
Name ==
"sse2.pminu.b" ||
Name ==
"sse41.pminuw" ||
2706 Name ==
"sse41.pminud" ||
Name.starts_with(
"avx2.pminu") ||
2707 Name.starts_with(
"avx512.mask.pminu")) {
2709 }
else if (
Name ==
"sse2.pmulu.dq" ||
Name ==
"avx2.pmulu.dq" ||
2710 Name ==
"avx512.pmulu.dq.512" ||
2711 Name.starts_with(
"avx512.mask.pmulu.dq.")) {
2713 }
else if (
Name ==
"sse41.pmuldq" ||
Name ==
"avx2.pmul.dq" ||
2714 Name ==
"avx512.pmul.dq.512" ||
2715 Name.starts_with(
"avx512.mask.pmul.dq.")) {
2717 }
else if (
Name ==
"sse.cvtsi2ss" ||
Name ==
"sse2.cvtsi2sd" ||
2718 Name ==
"sse.cvtsi642ss" ||
Name ==
"sse2.cvtsi642sd") {
2721 cast<VectorType>(CI->
getType())->getElementType());
2723 }
else if (
Name ==
"avx512.cvtusi2sd") {
2726 cast<VectorType>(CI->
getType())->getElementType());
2728 }
else if (
Name ==
"sse2.cvtss2sd") {
2731 Rep, cast<VectorType>(CI->
getType())->getElementType());
2733 }
else if (
Name ==
"sse2.cvtdq2pd" ||
Name ==
"sse2.cvtdq2ps" ||
2734 Name ==
"avx.cvtdq2.pd.256" ||
Name ==
"avx.cvtdq2.ps.256" ||
2735 Name.starts_with(
"avx512.mask.cvtdq2pd.") ||
2736 Name.starts_with(
"avx512.mask.cvtudq2pd.") ||
2737 Name.starts_with(
"avx512.mask.cvtdq2ps.") ||
2738 Name.starts_with(
"avx512.mask.cvtudq2ps.") ||
2739 Name.starts_with(
"avx512.mask.cvtqq2pd.") ||
2740 Name.starts_with(
"avx512.mask.cvtuqq2pd.") ||
2741 Name ==
"avx512.mask.cvtqq2ps.256" ||
2742 Name ==
"avx512.mask.cvtqq2ps.512" ||
2743 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2744 Name ==
"avx512.mask.cvtuqq2ps.512" ||
Name ==
"sse2.cvtps2pd" ||
2745 Name ==
"avx.cvt.ps2.pd.256" ||
2746 Name ==
"avx512.mask.cvtps2pd.128" ||
2747 Name ==
"avx512.mask.cvtps2pd.256") {
2748 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2750 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2752 unsigned NumDstElts = DstTy->getNumElements();
2754 assert(NumDstElts == 2 &&
"Unexpected vector size");
2758 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2759 bool IsUnsigned =
Name.contains(
"cvtu");
2761 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtps2pd");
2764 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2765 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2766 : Intrinsic::x86_avx512_sitofp_round;
2770 Rep = IsUnsigned ? Builder.
CreateUIToFP(Rep, DstTy,
"cvt")
2777 }
else if (
Name.starts_with(
"avx512.mask.vcvtph2ps.") ||
2778 Name.starts_with(
"vcvtph2ps.")) {
2779 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2781 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2782 unsigned NumDstElts = DstTy->getNumElements();
2783 if (NumDstElts != SrcTy->getNumElements()) {
2784 assert(NumDstElts == 4 &&
"Unexpected vector size");
2789 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtph2ps");
2793 }
else if (
Name.starts_with(
"avx512.mask.load")) {
2798 }
else if (
Name.starts_with(
"avx512.mask.expand.load.")) {
2799 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2801 ResultTy->getNumElements());
2804 Intrinsic::masked_expandload, ResultTy,
2806 }
else if (
Name.starts_with(
"avx512.mask.compress.store.")) {
2810 cast<FixedVectorType>(ResultTy)->getNumElements());
2813 Intrinsic::masked_compressstore, ResultTy,
2815 }
else if (
Name.starts_with(
"avx512.mask.compress.") ||
2816 Name.starts_with(
"avx512.mask.expand.")) {
2817 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2820 ResultTy->getNumElements());
2822 bool IsCompress =
Name[12] ==
'c';
2823 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2824 : Intrinsic::x86_avx512_mask_expand;
2827 }
else if (
Name.starts_with(
"xop.vpcom")) {
2829 if (
Name.ends_with(
"ub") ||
Name.ends_with(
"uw") ||
Name.ends_with(
"ud") ||
2830 Name.ends_with(
"uq"))
2832 else if (
Name.ends_with(
"b") ||
Name.ends_with(
"w") ||
2833 Name.ends_with(
"d") ||
Name.ends_with(
"q"))
2840 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2843 if (
Name.starts_with(
"lt"))
2845 else if (
Name.starts_with(
"le"))
2847 else if (
Name.starts_with(
"gt"))
2849 else if (
Name.starts_with(
"ge"))
2851 else if (
Name.starts_with(
"eq"))
2853 else if (
Name.starts_with(
"ne"))
2855 else if (
Name.starts_with(
"false"))
2857 else if (
Name.starts_with(
"true"))
2864 }
else if (
Name.starts_with(
"xop.vpcmov")) {
2869 Rep = Builder.
CreateOr(Sel0, Sel1);
2870 }
else if (
Name.starts_with(
"xop.vprot") ||
Name.starts_with(
"avx512.prol") ||
2871 Name.starts_with(
"avx512.mask.prol")) {
2873 }
else if (
Name.starts_with(
"avx512.pror") ||
2874 Name.starts_with(
"avx512.mask.pror")) {
2876 }
else if (
Name.starts_with(
"avx512.vpshld.") ||
2877 Name.starts_with(
"avx512.mask.vpshld") ||
2878 Name.starts_with(
"avx512.maskz.vpshld")) {
2879 bool ZeroMask =
Name[11] ==
'z';
2881 }
else if (
Name.starts_with(
"avx512.vpshrd.") ||
2882 Name.starts_with(
"avx512.mask.vpshrd") ||
2883 Name.starts_with(
"avx512.maskz.vpshrd")) {
2884 bool ZeroMask =
Name[11] ==
'z';
2886 }
else if (
Name ==
"sse42.crc32.64.8") {
2892 }
else if (
Name.starts_with(
"avx.vbroadcast.s") ||
2893 Name.starts_with(
"avx512.vbroadcast.s")) {
2895 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2896 Type *EltTy = VecTy->getElementType();
2897 unsigned EltNum = VecTy->getNumElements();
2901 for (
unsigned I = 0;
I < EltNum; ++
I)
2903 }
else if (
Name.starts_with(
"sse41.pmovsx") ||
2904 Name.starts_with(
"sse41.pmovzx") ||
2905 Name.starts_with(
"avx2.pmovsx") ||
2906 Name.starts_with(
"avx2.pmovzx") ||
2907 Name.starts_with(
"avx512.mask.pmovsx") ||
2908 Name.starts_with(
"avx512.mask.pmovzx")) {
2909 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2910 unsigned NumDstElts = DstTy->getNumElements();
2914 for (
unsigned i = 0; i != NumDstElts; ++i)
2919 bool DoSext =
Name.contains(
"pmovsx");
2926 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
2927 Name ==
"avx512.mask.pmov.qd.512" ||
2928 Name ==
"avx512.mask.pmov.wb.256" ||
2929 Name ==
"avx512.mask.pmov.wb.512") {
2934 }
else if (
Name.starts_with(
"avx.vbroadcastf128") ||
2935 Name ==
"avx2.vbroadcasti128") {
2937 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
2941 if (NumSrcElts == 2)
2946 }
else if (
Name.starts_with(
"avx512.mask.shuf.i") ||
2947 Name.starts_with(
"avx512.mask.shuf.f")) {
2948 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2952 unsigned ControlBitsMask = NumLanes - 1;
2953 unsigned NumControlBits = NumLanes / 2;
2956 for (
unsigned l = 0; l != NumLanes; ++l) {
2957 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2959 if (l >= NumLanes / 2)
2960 LaneMask += NumLanes;
2961 for (
unsigned i = 0; i != NumElementsInLane; ++i)
2962 ShuffleMask.
push_back(LaneMask * NumElementsInLane + i);
2968 }
else if (
Name.starts_with(
"avx512.mask.broadcastf") ||
2969 Name.starts_with(
"avx512.mask.broadcasti")) {
2972 unsigned NumDstElts =
2973 cast<FixedVectorType>(CI->
getType())->getNumElements();
2976 for (
unsigned i = 0; i != NumDstElts; ++i)
2977 ShuffleMask[i] = i % NumSrcElts;
2983 }
else if (
Name.starts_with(
"avx2.pbroadcast") ||
2984 Name.starts_with(
"avx2.vbroadcast") ||
2985 Name.starts_with(
"avx512.pbroadcast") ||
2986 Name.starts_with(
"avx512.mask.broadcast.s")) {
2998 }
else if (
Name.starts_with(
"sse2.padds.") ||
2999 Name.starts_with(
"avx2.padds.") ||
3000 Name.starts_with(
"avx512.padds.") ||
3001 Name.starts_with(
"avx512.mask.padds.")) {
3003 }
else if (
Name.starts_with(
"sse2.psubs.") ||
3004 Name.starts_with(
"avx2.psubs.") ||
3005 Name.starts_with(
"avx512.psubs.") ||
3006 Name.starts_with(
"avx512.mask.psubs.")) {
3008 }
else if (
Name.starts_with(
"sse2.paddus.") ||
3009 Name.starts_with(
"avx2.paddus.") ||
3010 Name.starts_with(
"avx512.mask.paddus.")) {
3012 }
else if (
Name.starts_with(
"sse2.psubus.") ||
3013 Name.starts_with(
"avx2.psubus.") ||
3014 Name.starts_with(
"avx512.mask.psubus.")) {
3016 }
else if (
Name.starts_with(
"avx512.mask.palignr.")) {
3021 }
else if (
Name.starts_with(
"avx512.mask.valign.")) {
3025 }
else if (
Name ==
"sse2.psll.dq" ||
Name ==
"avx2.psll.dq") {
3027 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3030 }
else if (
Name ==
"sse2.psrl.dq" ||
Name ==
"avx2.psrl.dq") {
3032 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3035 }
else if (
Name ==
"sse2.psll.dq.bs" ||
Name ==
"avx2.psll.dq.bs" ||
3036 Name ==
"avx512.psll.dq.512") {
3038 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3040 }
else if (
Name ==
"sse2.psrl.dq.bs" ||
Name ==
"avx2.psrl.dq.bs" ||
3041 Name ==
"avx512.psrl.dq.512") {
3043 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3045 }
else if (
Name ==
"sse41.pblendw" ||
Name.starts_with(
"sse41.blendp") ||
3046 Name.starts_with(
"avx.blend.p") ||
Name ==
"avx2.pblendw" ||
3047 Name.starts_with(
"avx2.pblendd.")) {
3050 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3051 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3052 unsigned NumElts = VecTy->getNumElements();
3055 for (
unsigned i = 0; i != NumElts; ++i)
3056 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3059 }
else if (
Name.starts_with(
"avx.vinsertf128.") ||
3060 Name ==
"avx2.vinserti128" ||
3061 Name.starts_with(
"avx512.mask.insert")) {
3064 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3065 unsigned DstNumElts =
3066 cast<FixedVectorType>(CI->
getType())->getNumElements();
3067 unsigned SrcNumElts =
3068 cast<FixedVectorType>(Op1->
getType())->getNumElements();
3069 unsigned Scale = DstNumElts / SrcNumElts;
3076 for (
unsigned i = 0; i != SrcNumElts; ++i)
3078 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
3079 Idxs[i] = SrcNumElts;
3094 for (
unsigned i = 0; i != DstNumElts; ++i)
3097 for (
unsigned i = 0; i != SrcNumElts; ++i)
3098 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3105 }
else if (
Name.starts_with(
"avx.vextractf128.") ||
3106 Name ==
"avx2.vextracti128" ||
3107 Name.starts_with(
"avx512.mask.vextract")) {
3109 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3110 unsigned DstNumElts =
3111 cast<FixedVectorType>(CI->
getType())->getNumElements();
3112 unsigned SrcNumElts =
3113 cast<FixedVectorType>(Op0->
getType())->getNumElements();
3114 unsigned Scale = SrcNumElts / DstNumElts;
3121 for (
unsigned i = 0; i != DstNumElts; ++i) {
3122 Idxs[i] = i + (Imm * DstNumElts);
3130 }
else if (
Name.starts_with(
"avx512.mask.perm.df.") ||
3131 Name.starts_with(
"avx512.mask.perm.di.")) {
3133 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3134 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3135 unsigned NumElts = VecTy->getNumElements();
3138 for (
unsigned i = 0; i != NumElts; ++i)
3139 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3146 }
else if (
Name.starts_with(
"avx.vperm2f128.") ||
Name ==
"avx2.vperm2i128") {
3157 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3158 unsigned HalfSize = NumElts / 2;
3170 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3171 for (
unsigned i = 0; i < HalfSize; ++i)
3172 ShuffleMask[i] = StartIndex + i;
3175 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3176 for (
unsigned i = 0; i < HalfSize; ++i)
3177 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3181 }
else if (
Name.starts_with(
"avx.vpermil.") ||
Name ==
"sse2.pshuf.d" ||
3182 Name.starts_with(
"avx512.mask.vpermil.p") ||
3183 Name.starts_with(
"avx512.mask.pshuf.d.")) {
3185 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3186 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3187 unsigned NumElts = VecTy->getNumElements();
3189 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3190 unsigned IdxMask = ((1 << IdxSize) - 1);
3196 for (
unsigned i = 0; i != NumElts; ++i)
3197 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3204 }
else if (
Name ==
"sse2.pshufl.w" ||
3205 Name.starts_with(
"avx512.mask.pshufl.w.")) {
3207 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3208 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3211 for (
unsigned l = 0; l != NumElts; l += 8) {
3212 for (
unsigned i = 0; i != 4; ++i)
3213 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3214 for (
unsigned i = 4; i != 8; ++i)
3215 Idxs[i + l] = i + l;
3223 }
else if (
Name ==
"sse2.pshufh.w" ||
3224 Name.starts_with(
"avx512.mask.pshufh.w.")) {
3226 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3227 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3230 for (
unsigned l = 0; l != NumElts; l += 8) {
3231 for (
unsigned i = 0; i != 4; ++i)
3232 Idxs[i + l] = i + l;
3233 for (
unsigned i = 0; i != 4; ++i)
3234 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3242 }
else if (
Name.starts_with(
"avx512.mask.shuf.p")) {
3245 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3246 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3249 unsigned HalfLaneElts = NumLaneElts / 2;
3252 for (
unsigned i = 0; i != NumElts; ++i) {
3254 Idxs[i] = i - (i % NumLaneElts);
3256 if ((i % NumLaneElts) >= HalfLaneElts)
3260 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3267 }
else if (
Name.starts_with(
"avx512.mask.movddup") ||
3268 Name.starts_with(
"avx512.mask.movshdup") ||
3269 Name.starts_with(
"avx512.mask.movsldup")) {
3271 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3275 if (
Name.starts_with(
"avx512.mask.movshdup."))
3279 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
3280 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
3281 Idxs[i + l + 0] = i + l +
Offset;
3282 Idxs[i + l + 1] = i + l +
Offset;
3289 }
else if (
Name.starts_with(
"avx512.mask.punpckl") ||
3290 Name.starts_with(
"avx512.mask.unpckl.")) {
3293 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3297 for (
int l = 0; l != NumElts; l += NumLaneElts)
3298 for (
int i = 0; i != NumLaneElts; ++i)
3299 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3305 }
else if (
Name.starts_with(
"avx512.mask.punpckh") ||
3306 Name.starts_with(
"avx512.mask.unpckh.")) {
3309 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3313 for (
int l = 0; l != NumElts; l += NumLaneElts)
3314 for (
int i = 0; i != NumLaneElts; ++i)
3315 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3321 }
else if (
Name.starts_with(
"avx512.mask.and.") ||
3322 Name.starts_with(
"avx512.mask.pand.")) {
3324 VectorType *ITy = VectorType::getInteger(FTy);
3330 }
else if (
Name.starts_with(
"avx512.mask.andn.") ||
3331 Name.starts_with(
"avx512.mask.pandn.")) {
3333 VectorType *ITy = VectorType::getInteger(FTy);
3340 }
else if (
Name.starts_with(
"avx512.mask.or.") ||
3341 Name.starts_with(
"avx512.mask.por.")) {
3343 VectorType *ITy = VectorType::getInteger(FTy);
3349 }
else if (
Name.starts_with(
"avx512.mask.xor.") ||
3350 Name.starts_with(
"avx512.mask.pxor.")) {
3352 VectorType *ITy = VectorType::getInteger(FTy);
3358 }
else if (
Name.starts_with(
"avx512.mask.padd.")) {
3362 }
else if (
Name.starts_with(
"avx512.mask.psub.")) {
3366 }
else if (
Name.starts_with(
"avx512.mask.pmull.")) {
3370 }
else if (
Name.starts_with(
"avx512.mask.add.p")) {
3371 if (
Name.ends_with(
".512")) {
3373 if (
Name[17] ==
's')
3374 IID = Intrinsic::x86_avx512_add_ps_512;
3376 IID = Intrinsic::x86_avx512_add_pd_512;
3386 }
else if (
Name.starts_with(
"avx512.mask.div.p")) {
3387 if (
Name.ends_with(
".512")) {
3389 if (
Name[17] ==
's')
3390 IID = Intrinsic::x86_avx512_div_ps_512;
3392 IID = Intrinsic::x86_avx512_div_pd_512;
3402 }
else if (
Name.starts_with(
"avx512.mask.mul.p")) {
3403 if (
Name.ends_with(
".512")) {
3405 if (
Name[17] ==
's')
3406 IID = Intrinsic::x86_avx512_mul_ps_512;
3408 IID = Intrinsic::x86_avx512_mul_pd_512;
3418 }
else if (
Name.starts_with(
"avx512.mask.sub.p")) {
3419 if (
Name.ends_with(
".512")) {
3421 if (
Name[17] ==
's')
3422 IID = Intrinsic::x86_avx512_sub_ps_512;
3424 IID = Intrinsic::x86_avx512_sub_pd_512;
3434 }
else if ((
Name.starts_with(
"avx512.mask.max.p") ||
3435 Name.starts_with(
"avx512.mask.min.p")) &&
3436 Name.drop_front(18) ==
".512") {
3437 bool IsDouble =
Name[17] ==
'd';
3438 bool IsMin =
Name[13] ==
'i';
3440 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
3441 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
3449 }
else if (
Name.starts_with(
"avx512.mask.lzcnt.")) {
3452 {CI->getArgOperand(0), Builder.getInt1(false)});
3455 }
else if (
Name.starts_with(
"avx512.mask.psll")) {
3456 bool IsImmediate =
Name[16] ==
'i' || (
Name.size() > 18 &&
Name[18] ==
'i');
3457 bool IsVariable =
Name[16] ==
'v';
3464 if (IsVariable &&
Name[17] !=
'.') {
3465 if (
Size ==
'd' &&
Name[17] ==
'2')
3466 IID = Intrinsic::x86_avx2_psllv_q;
3467 else if (
Size ==
'd' &&
Name[17] ==
'4')
3468 IID = Intrinsic::x86_avx2_psllv_q_256;
3469 else if (
Size ==
's' &&
Name[17] ==
'4')
3470 IID = Intrinsic::x86_avx2_psllv_d;
3471 else if (
Size ==
's' &&
Name[17] ==
'8')
3472 IID = Intrinsic::x86_avx2_psllv_d_256;
3473 else if (
Size ==
'h' &&
Name[17] ==
'8')
3474 IID = Intrinsic::x86_avx512_psllv_w_128;
3475 else if (
Size ==
'h' &&
Name[17] ==
'1')
3476 IID = Intrinsic::x86_avx512_psllv_w_256;
3477 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3478 IID = Intrinsic::x86_avx512_psllv_w_512;
3481 }
else if (
Name.ends_with(
".128")) {
3483 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3484 : Intrinsic::x86_sse2_psll_d;
3485 else if (
Size ==
'q')
3486 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3487 : Intrinsic::x86_sse2_psll_q;
3488 else if (
Size ==
'w')
3489 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3490 : Intrinsic::x86_sse2_psll_w;
3493 }
else if (
Name.ends_with(
".256")) {
3495 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3496 : Intrinsic::x86_avx2_psll_d;
3497 else if (
Size ==
'q')
3498 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3499 : Intrinsic::x86_avx2_psll_q;
3500 else if (
Size ==
'w')
3501 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3502 : Intrinsic::x86_avx2_psll_w;
3507 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
3508 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
3509 : Intrinsic::x86_avx512_psll_d_512;
3510 else if (
Size ==
'q')
3511 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
3512 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
3513 : Intrinsic::x86_avx512_psll_q_512;
3514 else if (
Size ==
'w')
3515 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3516 : Intrinsic::x86_avx512_psll_w_512;
3522 }
else if (
Name.starts_with(
"avx512.mask.psrl")) {
3523 bool IsImmediate =
Name[16] ==
'i' || (
Name.size() > 18 &&
Name[18] ==
'i');
3524 bool IsVariable =
Name[16] ==
'v';
3531 if (IsVariable &&
Name[17] !=
'.') {
3532 if (
Size ==
'd' &&
Name[17] ==
'2')
3533 IID = Intrinsic::x86_avx2_psrlv_q;
3534 else if (
Size ==
'd' &&
Name[17] ==
'4')
3535 IID = Intrinsic::x86_avx2_psrlv_q_256;
3536 else if (
Size ==
's' &&
Name[17] ==
'4')
3537 IID = Intrinsic::x86_avx2_psrlv_d;
3538 else if (
Size ==
's' &&
Name[17] ==
'8')
3539 IID = Intrinsic::x86_avx2_psrlv_d_256;
3540 else if (
Size ==
'h' &&
Name[17] ==
'8')
3541 IID = Intrinsic::x86_avx512_psrlv_w_128;
3542 else if (
Size ==
'h' &&
Name[17] ==
'1')
3543 IID = Intrinsic::x86_avx512_psrlv_w_256;
3544 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3545 IID = Intrinsic::x86_avx512_psrlv_w_512;
3548 }
else if (
Name.ends_with(
".128")) {
3550 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3551 : Intrinsic::x86_sse2_psrl_d;
3552 else if (
Size ==
'q')
3553 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3554 : Intrinsic::x86_sse2_psrl_q;
3555 else if (
Size ==
'w')
3556 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3557 : Intrinsic::x86_sse2_psrl_w;
3560 }
else if (
Name.ends_with(
".256")) {
3562 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3563 : Intrinsic::x86_avx2_psrl_d;
3564 else if (
Size ==
'q')
3565 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3566 : Intrinsic::x86_avx2_psrl_q;
3567 else if (
Size ==
'w')
3568 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3569 : Intrinsic::x86_avx2_psrl_w;
3574 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
3575 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
3576 : Intrinsic::x86_avx512_psrl_d_512;
3577 else if (
Size ==
'q')
3578 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
3579 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
3580 : Intrinsic::x86_avx512_psrl_q_512;
3581 else if (
Size ==
'w')
3582 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3583 : Intrinsic::x86_avx512_psrl_w_512;
3589 }
else if (
Name.starts_with(
"avx512.mask.psra")) {
3590 bool IsImmediate =
Name[16] ==
'i' || (
Name.size() > 18 &&
Name[18] ==
'i');
3591 bool IsVariable =
Name[16] ==
'v';
3598 if (IsVariable &&
Name[17] !=
'.') {
3599 if (
Size ==
's' &&
Name[17] ==
'4')
3600 IID = Intrinsic::x86_avx2_psrav_d;
3601 else if (
Size ==
's' &&
Name[17] ==
'8')
3602 IID = Intrinsic::x86_avx2_psrav_d_256;
3603 else if (
Size ==
'h' &&
Name[17] ==
'8')
3604 IID = Intrinsic::x86_avx512_psrav_w_128;
3605 else if (
Size ==
'h' &&
Name[17] ==
'1')
3606 IID = Intrinsic::x86_avx512_psrav_w_256;
3607 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3608 IID = Intrinsic::x86_avx512_psrav_w_512;
3611 }
else if (
Name.ends_with(
".128")) {
3613 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3614 : Intrinsic::x86_sse2_psra_d;
3615 else if (
Size ==
'q')
3616 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
3617 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
3618 : Intrinsic::x86_avx512_psra_q_128;
3619 else if (
Size ==
'w')
3620 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3621 : Intrinsic::x86_sse2_psra_w;
3624 }
else if (
Name.ends_with(
".256")) {
3626 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3627 : Intrinsic::x86_avx2_psra_d;
3628 else if (
Size ==
'q')
3629 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
3630 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
3631 : Intrinsic::x86_avx512_psra_q_256;
3632 else if (
Size ==
'w')
3633 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3634 : Intrinsic::x86_avx2_psra_w;
3639 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
3640 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
3641 : Intrinsic::x86_avx512_psra_d_512;
3642 else if (
Size ==
'q')
3643 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
3644 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
3645 : Intrinsic::x86_avx512_psra_q_512;
3646 else if (
Size ==
'w')
3647 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3648 : Intrinsic::x86_avx512_psra_w_512;
3654 }
else if (
Name.starts_with(
"avx512.mask.move.s")) {
3656 }
else if (
Name.starts_with(
"avx512.cvtmask2")) {
3658 }
else if (
Name.ends_with(
".movntdqa")) {
3667 }
else if (
Name.starts_with(
"fma.vfmadd.") ||
3668 Name.starts_with(
"fma.vfmsub.") ||
3669 Name.starts_with(
"fma.vfnmadd.") ||
3670 Name.starts_with(
"fma.vfnmsub.")) {
3671 bool NegMul =
Name[6] ==
'n';
3672 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3673 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3684 if (NegMul && !IsScalar)
3686 if (NegMul && IsScalar)
3695 }
else if (
Name.starts_with(
"fma4.vfmadd.s")) {
3707 }
else if (
Name.starts_with(
"avx512.mask.vfmadd.s") ||
3708 Name.starts_with(
"avx512.maskz.vfmadd.s") ||
3709 Name.starts_with(
"avx512.mask3.vfmadd.s") ||
3710 Name.starts_with(
"avx512.mask3.vfmsub.s") ||
3711 Name.starts_with(
"avx512.mask3.vfnmsub.s")) {
3712 bool IsMask3 =
Name[11] ==
'3';
3713 bool IsMaskZ =
Name[11] ==
'z';
3715 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3716 bool NegMul =
Name[2] ==
'n';
3717 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3723 if (NegMul && (IsMask3 || IsMaskZ))
3725 if (NegMul && !(IsMask3 || IsMaskZ))
3735 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3739 if (
Name.back() ==
'd')
3740 IID = Intrinsic::x86_avx512_vfmadd_f64;
3742 IID = Intrinsic::x86_avx512_vfmadd_f32;
3754 if (NegAcc && IsMask3)
3761 }
else if (
Name.starts_with(
"avx512.mask.vfmadd.p") ||
3762 Name.starts_with(
"avx512.mask.vfnmadd.p") ||
3763 Name.starts_with(
"avx512.mask.vfnmsub.p") ||
3764 Name.starts_with(
"avx512.mask3.vfmadd.p") ||
3765 Name.starts_with(
"avx512.mask3.vfmsub.p") ||
3766 Name.starts_with(
"avx512.mask3.vfnmsub.p") ||
3767 Name.starts_with(
"avx512.maskz.vfmadd.p")) {
3768 bool IsMask3 =
Name[11] ==
'3';
3769 bool IsMaskZ =
Name[11] ==
'z';
3771 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3772 bool NegMul =
Name[2] ==
'n';
3773 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3779 if (NegMul && (IsMask3 || IsMaskZ))
3781 if (NegMul && !(IsMask3 || IsMaskZ))
3788 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
3792 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3794 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3806 }
else if (
Name.starts_with(
"fma.vfmsubadd.p")) {
3810 if (VecWidth == 128 && EltWidth == 32)
3811 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3812 else if (VecWidth == 256 && EltWidth == 32)
3813 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3814 else if (VecWidth == 128 && EltWidth == 64)
3815 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3816 else if (VecWidth == 256 && EltWidth == 64)
3817 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3825 }
else if (
Name.starts_with(
"avx512.mask.vfmaddsub.p") ||
3826 Name.starts_with(
"avx512.mask3.vfmaddsub.p") ||
3827 Name.starts_with(
"avx512.maskz.vfmaddsub.p") ||
3828 Name.starts_with(
"avx512.mask3.vfmsubadd.p")) {
3829 bool IsMask3 =
Name[11] ==
'3';
3830 bool IsMaskZ =
Name[11] ==
'z';
3832 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3833 bool IsSubAdd =
Name[3] ==
's';
3838 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3840 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3849 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3864 for (
int i = 0; i != NumElts; ++i)
3865 Idxs[i] = i + (i % 2) * NumElts;
3875 }
else if (
Name.starts_with(
"avx512.mask.pternlog.") ||
3876 Name.starts_with(
"avx512.maskz.pternlog.")) {
3877 bool ZeroMask =
Name[11] ==
'z';
3881 if (VecWidth == 128 && EltWidth == 32)
3882 IID = Intrinsic::x86_avx512_pternlog_d_128;
3883 else if (VecWidth == 256 && EltWidth == 32)
3884 IID = Intrinsic::x86_avx512_pternlog_d_256;
3885 else if (VecWidth == 512 && EltWidth == 32)
3886 IID = Intrinsic::x86_avx512_pternlog_d_512;
3887 else if (VecWidth == 128 && EltWidth == 64)
3888 IID = Intrinsic::x86_avx512_pternlog_q_128;
3889 else if (VecWidth == 256 && EltWidth == 64)
3890 IID = Intrinsic::x86_avx512_pternlog_q_256;
3891 else if (VecWidth == 512 && EltWidth == 64)
3892 IID = Intrinsic::x86_avx512_pternlog_q_512;
3902 }
else if (
Name.starts_with(
"avx512.mask.vpmadd52") ||
3903 Name.starts_with(
"avx512.maskz.vpmadd52")) {
3904 bool ZeroMask =
Name[11] ==
'z';
3908 if (VecWidth == 128 && !
High)
3909 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3910 else if (VecWidth == 256 && !
High)
3911 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3912 else if (VecWidth == 512 && !
High)
3913 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3914 else if (VecWidth == 128 &&
High)
3915 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3916 else if (VecWidth == 256 &&
High)
3917 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3918 else if (VecWidth == 512 &&
High)
3919 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3929 }
else if (
Name.starts_with(
"avx512.mask.vpermi2var.") ||
3930 Name.starts_with(
"avx512.mask.vpermt2var.") ||
3931 Name.starts_with(
"avx512.maskz.vpermt2var.")) {
3932 bool ZeroMask =
Name[11] ==
'z';
3933 bool IndexForm =
Name[17] ==
'i';
3935 }
else if (
Name.starts_with(
"avx512.mask.vpdpbusd.") ||
3936 Name.starts_with(
"avx512.maskz.vpdpbusd.") ||
3937 Name.starts_with(
"avx512.mask.vpdpbusds.") ||
3938 Name.starts_with(
"avx512.maskz.vpdpbusds.")) {
3939 bool ZeroMask =
Name[11] ==
'z';
3940 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3943 if (VecWidth == 128 && !IsSaturating)
3944 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3945 else if (VecWidth == 256 && !IsSaturating)
3946 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3947 else if (VecWidth == 512 && !IsSaturating)
3948 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3949 else if (VecWidth == 128 && IsSaturating)
3950 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3951 else if (VecWidth == 256 && IsSaturating)
3952 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3953 else if (VecWidth == 512 && IsSaturating)
3954 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3964 }
else if (
Name.starts_with(
"avx512.mask.vpdpwssd.") ||
3965 Name.starts_with(
"avx512.maskz.vpdpwssd.") ||
3966 Name.starts_with(
"avx512.mask.vpdpwssds.") ||
3967 Name.starts_with(
"avx512.maskz.vpdpwssds.")) {
3968 bool ZeroMask =
Name[11] ==
'z';
3969 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3972 if (VecWidth == 128 && !IsSaturating)
3973 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3974 else if (VecWidth == 256 && !IsSaturating)
3975 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3976 else if (VecWidth == 512 && !IsSaturating)
3977 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3978 else if (VecWidth == 128 && IsSaturating)
3979 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3980 else if (VecWidth == 256 && IsSaturating)
3981 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3982 else if (VecWidth == 512 && IsSaturating)
3983 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3993 }
else if (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
3994 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
3995 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64") {
3997 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
3998 IID = Intrinsic::x86_addcarry_32;
3999 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
4000 IID = Intrinsic::x86_addcarry_64;
4001 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
4002 IID = Intrinsic::x86_subborrow_32;
4003 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
4004 IID = Intrinsic::x86_subborrow_64;
4021 }
else if (
Name.starts_with(
"avx512.mask.") &&
4033 .
Case(
"sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4034 .
Case(
"sve.fcvtnt.bf16f32", Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4046 if (Args[1]->
getType() != BadPredTy)
4049 Args[1] = Builder.
CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4050 BadPredTy, Args[1]);
4051 Args[1] = Builder.
CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
4052 GoodPredTy, Args[1]);
4060 if (
Name ==
"mve.vctp64.old") {
4067 Intrinsic::arm_mve_pred_v2i,
4068 {VectorType::get(Builder.
getInt1Ty(), 2,
false)}, VCTP);
4070 Intrinsic::arm_mve_pred_i2v,
4071 {VectorType::get(Builder.
getInt1Ty(), 4,
false)}, C1);
4072 }
else if (
Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4073 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4074 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4075 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4077 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4078 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4079 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
4080 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
4082 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
4083 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
4084 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
4085 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
4086 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
4087 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
4088 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
4089 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
4090 std::vector<Type *> Tys;
4094 case Intrinsic::arm_mve_mull_int_predicated:
4095 case Intrinsic::arm_mve_vqdmull_predicated:
4096 case Intrinsic::arm_mve_vldr_gather_base_predicated:
4099 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4100 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4101 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4105 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4109 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4113 case Intrinsic::arm_cde_vcx1q_predicated:
4114 case Intrinsic::arm_cde_vcx1qa_predicated:
4115 case Intrinsic::arm_cde_vcx2q_predicated:
4116 case Intrinsic::arm_cde_vcx2qa_predicated:
4117 case Intrinsic::arm_cde_vcx3q_predicated:
4118 case Intrinsic::arm_cde_vcx3qa_predicated:
4125 std::vector<Value *> Ops;
4127 Type *Ty =
Op->getType();
4130 Intrinsic::arm_mve_pred_v2i,
4131 {VectorType::get(Builder.
getInt1Ty(), 4,
false)},
Op);
4165 if (NumOperands < 3)
4178 bool IsVolatile =
false;
4182 if (NumOperands > 3)
4187 if (NumOperands > 5) {
4189 IsVolatile = !VolatileArg || !VolatileArg->
isZero();
4195 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
4196 Order = AtomicOrdering::SequentiallyConsistent;
4203 if (VT->getElementType()->isIntegerTy(16)) {
4216 unsigned AddrSpace = PtrTy->getAddressSpace();
4219 RMW->
setMetadata(
"amdgpu.no.fine.grained.memory", EmptyMD);
4221 RMW->
setMetadata(
"amdgpu.ignore.denormal.mode", EmptyMD);
4226 MDNode *RangeNotPrivate =
4229 RMW->
setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
4239template <
typename MDType>
4242 return dyn_cast<MDType>(MAV->getMetadata());
4251 if (
Name ==
"label") {
4253 }
else if (
Name ==
"assign") {
4255 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
4256 unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3),
4257 unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5),
4259 }
else if (
Name ==
"declare") {
4261 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
4262 unwrapMAVOp<DIExpression>(CI, 2), CI->
getDebugLoc(),
4263 DbgVariableRecord::LocationType::Declare);
4264 }
else if (
Name ==
"addr") {
4269 unwrapMAVOp<DILocalVariable>(CI, 1), Expr,
4271 }
else if (
Name ==
"value") {
4274 unsigned ExprOp = 2;
4284 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp),
4285 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->
getDebugLoc());
4287 assert(DR &&
"Unhandled intrinsic kind in upgrade to DbgRecord");
4306 bool FallthroughToDefaultUpgrade =
false;
4310 assert(
Name.starts_with(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
4313 bool IsX86 =
Name.consume_front(
"x86.");
4314 bool IsNVVM =
Name.consume_front(
"nvvm.");
4315 bool IsAArch64 =
Name.consume_front(
"aarch64.");
4316 bool IsARM =
Name.consume_front(
"arm.");
4317 bool IsAMDGCN =
Name.consume_front(
"amdgcn.");
4318 bool IsDbg =
Name.consume_front(
"dbg.");
4319 Value *Rep =
nullptr;
4321 if (!IsX86 &&
Name ==
"stackprotectorcheck") {
4323 }
else if (IsNVVM) {
4327 }
else if (IsAArch64) {
4331 }
else if (IsAMDGCN) {
4343 FallthroughToDefaultUpgrade =
true;
4351 if (!FallthroughToDefaultUpgrade) {
4359 const auto &DefaultCase = [&]() ->
void {
4364 "Unknown function for CallBase upgrade and isn't just a name change");
4370 if (
auto *OldST = dyn_cast<StructType>(CI->
getType())) {
4372 "Return type must have changed");
4373 assert(OldST->getNumElements() ==
4374 cast<StructType>(NewFn->
getReturnType())->getNumElements() &&
4375 "Must have same number of elements");
4381 for (
unsigned Idx = 0;
Idx < OldST->getNumElements(); ++
Idx) {
4402 case Intrinsic::arm_neon_vst1:
4403 case Intrinsic::arm_neon_vst2:
4404 case Intrinsic::arm_neon_vst3:
4405 case Intrinsic::arm_neon_vst4:
4406 case Intrinsic::arm_neon_vst2lane:
4407 case Intrinsic::arm_neon_vst3lane:
4408 case Intrinsic::arm_neon_vst4lane: {
4413 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4414 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4415 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4419 cast<ConstantInt>(Args[3])->getZExtValue());
4423 case Intrinsic::aarch64_sve_ld3_sret:
4424 case Intrinsic::aarch64_sve_ld4_sret:
4425 case Intrinsic::aarch64_sve_ld2_sret: {
4433 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4434 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4438 for (
unsigned I = 0;
I <
N;
I++) {
4443 NewCall = dyn_cast<CallInst>(Ret);
4447 case Intrinsic::coro_end: {
4454 case Intrinsic::vector_extract: {
4457 if (!
Name.starts_with(
"aarch64.sve.tuple.get")) {
4461 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4462 unsigned MinElts =
RetTy->getMinNumElements();
4463 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4469 case Intrinsic::vector_insert: {
4472 if (!
Name.starts_with(
"aarch64.sve.tuple")) {
4476 if (
Name.starts_with(
"aarch64.sve.tuple.set")) {
4477 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4485 if (
Name.starts_with(
"aarch64.sve.tuple.create")) {
4491 assert(
N > 1 &&
"Create is expected to be between 2-4");
4492 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4494 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4495 for (
unsigned I = 0;
I <
N;
I++) {
4500 NewCall = dyn_cast<CallInst>(Ret);
4505 case Intrinsic::arm_neon_bfdot:
4506 case Intrinsic::arm_neon_bfmmla:
4507 case Intrinsic::arm_neon_bfmlalb:
4508 case Intrinsic::arm_neon_bfmlalt:
4509 case Intrinsic::aarch64_neon_bfdot:
4510 case Intrinsic::aarch64_neon_bfmmla:
4511 case Intrinsic::aarch64_neon_bfmlalb:
4512 case Intrinsic::aarch64_neon_bfmlalt: {
4515 "Mismatch between function args and call args");
4516 size_t OperandWidth =
4518 assert((OperandWidth == 64 || OperandWidth == 128) &&
4519 "Unexpected operand width");
4521 auto Iter = CI->
args().begin();
4522 Args.push_back(*Iter++);
4529 case Intrinsic::bitreverse:
4533 case Intrinsic::ctlz:
4534 case Intrinsic::cttz:
4536 "Mismatch between function args and call args");
4541 case Intrinsic::objectsize: {
4542 Value *NullIsUnknownSize =
4551 case Intrinsic::ctpop:
4555 case Intrinsic::convert_from_fp16:
4559 case Intrinsic::dbg_value: {
4563 if (
Name.starts_with(
"dbg.addr")) {
4565 cast<MetadataAsValue>(CI->
getArgOperand(2))->getMetadata());
4577 if (
Offset->isZeroValue()) {
4587 case Intrinsic::ptr_annotation:
4604 case Intrinsic::var_annotation:
4620 case Intrinsic::riscv_aes32dsi:
4621 case Intrinsic::riscv_aes32dsmi:
4622 case Intrinsic::riscv_aes32esi:
4623 case Intrinsic::riscv_aes32esmi:
4624 case Intrinsic::riscv_sm4ks:
4625 case Intrinsic::riscv_sm4ed: {
4640 cast<ConstantInt>(Arg2)->getZExtValue());
4642 NewCall = Builder.
CreateCall(NewFn, {Arg0, Arg1, Arg2});
4643 Value *Res = NewCall;
4651 case Intrinsic::riscv_sha256sig0:
4652 case Intrinsic::riscv_sha256sig1:
4653 case Intrinsic::riscv_sha256sum0:
4654 case Intrinsic::riscv_sha256sum1:
4655 case Intrinsic::riscv_sm3p0:
4656 case Intrinsic::riscv_sm3p1: {
4674 case Intrinsic::x86_xop_vfrcz_ss:
4675 case Intrinsic::x86_xop_vfrcz_sd:
4679 case Intrinsic::x86_xop_vpermil2pd:
4680 case Intrinsic::x86_xop_vpermil2ps:
4681 case Intrinsic::x86_xop_vpermil2pd_256:
4682 case Intrinsic::x86_xop_vpermil2ps_256: {
4685 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4691 case Intrinsic::x86_sse41_ptestc:
4692 case Intrinsic::x86_sse41_ptestz:
4693 case Intrinsic::x86_sse41_ptestnzc: {
4710 NewCall = Builder.
CreateCall(NewFn, {BC0, BC1});
4714 case Intrinsic::x86_rdtscp: {
4733 case Intrinsic::x86_sse41_insertps:
4734 case Intrinsic::x86_sse41_dppd:
4735 case Intrinsic::x86_sse41_dpps:
4736 case Intrinsic::x86_sse41_mpsadbw:
4737 case Intrinsic::x86_avx_dp_ps_256:
4738 case Intrinsic::x86_avx2_mpsadbw: {
4749 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4750 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4751 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4752 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4753 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4754 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4757 cast<FixedVectorType>(Args[0]->
getType())->getNumElements();
4769 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4770 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4771 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4772 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4773 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4774 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4776 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4778 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4791 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4792 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4793 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4796 cast<FixedVectorType>(CI->
getType())->getNumElements() * 2;
4806 case Intrinsic::thread_pointer: {
4811 case Intrinsic::memcpy:
4812 case Intrinsic::memmove:
4813 case Intrinsic::memset: {
4833 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4834 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4836 auto *MemCI = cast<MemIntrinsic>(NewCall);
4839 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
4841 if (
auto *MTI = dyn_cast<MemTransferInst>(MemCI))
4842 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
4846 assert(NewCall &&
"Should have either set this variable or returned through "
4847 "the default case");
4854 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
4863 if (
CallBase *CB = dyn_cast<CallBase>(U))
4867 F->eraseFromParent();
4873 if (NumOperands == 0)
4877 if (isa<MDNode>(MD.
getOperand(0)) && NumOperands >= 3)
4881 if (NumOperands == 3) {
4885 Metadata *Elts2[] = {ScalarType, ScalarType,
4899 if (Opc != Instruction::BitCast)
4903 Type *SrcTy = V->getType();
4920 if (Opc != Instruction::BitCast)
4923 Type *SrcTy =
C->getType();
4948 unsigned Version = 0;
4949 if (
NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
4950 auto OpIt =
find_if(ModFlags->operands(), [](
const MDNode *Flag) {
4951 if (Flag->getNumOperands() < 3)
4953 if (MDString *K = dyn_cast_or_null<MDString>(Flag->getOperand(1)))
4954 return K->getString() ==
"Debug Info Version";
4957 if (OpIt != ModFlags->op_end()) {
4958 const MDOperand &ValOp = (*OpIt)->getOperand(2);
4959 if (
auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
4960 Version = CI->getZExtValue();
4965 bool BrokenDebugInfo =
false;
4968 if (!BrokenDebugInfo)
4974 M.getContext().diagnose(Diag);
4981 M.getContext().diagnose(DiagVersion);
4989 bool Changed =
false;
4990 const char *MarkerKey =
"clang.arc.retainAutoreleasedReturnValueMarker";
4991 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4992 if (ModRetainReleaseMarker) {
4995 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(0));
4998 ID->getString().split(ValueComp,
"#");
4999 if (ValueComp.
size() == 2) {
5000 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
5004 M.eraseNamedMetadata(ModRetainReleaseMarker);
5015 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
5017 Function *Fn = M.getFunction(OldFunc);
5026 CallInst *CI = dyn_cast<CallInst>(U);
5041 bool InvalidCast =
false;
5043 for (
unsigned I = 0, E = CI->
arg_size();
I != E; ++
I) {
5048 if (I < NewFuncTy->getNumParams()) {
5058 Args.push_back(Arg);
5083 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
5091 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
5092 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
5093 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
5094 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
5095 {
"objc_autoreleaseReturnValue",
5096 llvm::Intrinsic::objc_autoreleaseReturnValue},
5097 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
5098 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
5099 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
5100 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
5101 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
5102 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
5103 {
"objc_release", llvm::Intrinsic::objc_release},
5104 {
"objc_retain", llvm::Intrinsic::objc_retain},
5105 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
5106 {
"objc_retainAutoreleaseReturnValue",
5107 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
5108 {
"objc_retainAutoreleasedReturnValue",
5109 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
5110 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
5111 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
5112 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
5113 {
"objc_unsafeClaimAutoreleasedReturnValue",
5114 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
5115 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
5116 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
5117 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
5118 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
5119 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
5120 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
5121 {
"objc_arc_annotation_topdown_bbstart",
5122 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
5123 {
"objc_arc_annotation_topdown_bbend",
5124 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
5125 {
"objc_arc_annotation_bottomup_bbstart",
5126 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
5127 {
"objc_arc_annotation_bottomup_bbend",
5128 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
5130 for (
auto &
I : RuntimeFuncs)
5131 UpgradeToIntrinsic(
I.first,
I.second);
5135 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
5139 bool HasObjCFlag =
false, HasClassProperties =
false, Changed =
false;
5140 bool HasSwiftVersionFlag =
false;
5141 uint8_t SwiftMajorVersion, SwiftMinorVersion;
5150 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(1));
5162 if (
ID->getString() ==
"Objective-C Image Info Version")
5164 if (
ID->getString() ==
"Objective-C Class Properties")
5165 HasClassProperties =
true;
5167 if (
ID->getString() ==
"PIC Level") {
5168 if (
auto *Behavior =
5169 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5170 uint64_t V = Behavior->getLimitedValue();
5176 if (
ID->getString() ==
"PIE Level")
5177 if (
auto *Behavior =
5178 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0)))
5184 if (
ID->getString() ==
"branch-target-enforcement" ||
5185 ID->getString().starts_with(
"sign-return-address")) {
5186 if (
auto *Behavior =
5187 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5192 Op->getOperand(1),
Op->getOperand(2)};
5202 if (
ID->getString() ==
"Objective-C Image Info Section") {
5203 if (
auto *
Value = dyn_cast_or_null<MDString>(
Op->getOperand(2))) {
5205 Value->getString().split(ValueComp,
" ");
5206 if (ValueComp.
size() != 1) {
5207 std::string NewValue;
5208 for (
auto &S : ValueComp)
5209 NewValue += S.str();
5210 Metadata *Ops[3] = {
Op->getOperand(0),
Op->getOperand(1),
5220 if (
ID->getString() ==
"Objective-C Garbage Collection") {
5221 auto Md = dyn_cast<ConstantAsMetadata>(
Op->getOperand(2));
5223 assert(Md->getValue() &&
"Expected non-empty metadata");
5224 auto Type = Md->getValue()->getType();
5227 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
5228 if ((Val & 0xff) != Val) {
5229 HasSwiftVersionFlag =
true;
5230 SwiftABIVersion = (Val & 0xff00) >> 8;
5231 SwiftMajorVersion = (Val & 0xff000000) >> 24;
5232 SwiftMinorVersion = (Val & 0xff0000) >> 16;
5243 if (
ID->getString() ==
"amdgpu_code_object_version") {
5246 MDString::get(M.getContext(),
"amdhsa_code_object_version"),
5258 if (HasObjCFlag && !HasClassProperties) {
5264 if (HasSwiftVersionFlag) {
5268 ConstantInt::get(Int8Ty, SwiftMajorVersion));
5270 ConstantInt::get(Int8Ty, SwiftMinorVersion));
5278 auto TrimSpaces = [](
StringRef Section) -> std::string {
5280 Section.split(Components,
',');
5285 for (
auto Component : Components)
5286 OS <<
',' << Component.trim();
5288 return std::string(
OS.str().substr(1));
5291 for (
auto &GV : M.globals()) {
5292 if (!GV.hasSection())
5297 if (!Section.starts_with(
"__DATA, __objc_catlist"))
5302 GV.setSection(TrimSpaces(Section));
5318struct StrictFPUpgradeVisitor :
public InstVisitor<StrictFPUpgradeVisitor> {
5319 StrictFPUpgradeVisitor() =
default;
5322 if (!Call.isStrictFP())
5324 if (isa<ConstrainedFPIntrinsic>(&Call))
5328 Call.removeFnAttr(Attribute::StrictFP);
5329 Call.addFnAttr(Attribute::NoBuiltin);
5334struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
5335 :
public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
5336 AMDGPUUnsafeFPAtomicsUpgradeVisitor() =
default;
5343 RMW.
setMetadata(
"amdgpu.no.fine.grained.host.memory", Empty);
5344 RMW.
setMetadata(
"amdgpu.no.remote.memory.access", Empty);
5345 RMW.
setMetadata(
"amdgpu.ignore.denormal.mode", Empty);
5353 if (!
F.isDeclaration() && !
F.hasFnAttribute(Attribute::StrictFP)) {
5354 StrictFPUpgradeVisitor SFPV;
5360 F.getReturnType(),
F.getAttributes().getRetAttrs()));
5361 for (
auto &Arg :
F.args())
5367 if (
Attribute A =
F.getFnAttribute(
"implicit-section-name");
5368 A.isValid() &&
A.isStringAttribute()) {
5369 F.setSection(
A.getValueAsString());
5370 F.removeFnAttr(
"implicit-section-name");
5377 if (
Attribute A =
F.getFnAttribute(
"amdgpu-unsafe-fp-atomics");
5380 if (
A.getValueAsBool()) {
5381 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
5387 F.removeFnAttr(
"amdgpu-unsafe-fp-atomics");
5393 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5396 if (
T->getNumOperands() < 1)
5398 auto *S = dyn_cast_or_null<MDString>(
T->getOperand(0));
5401 return S->getString().starts_with(
"llvm.vectorizer.");
5405 StringRef OldPrefix =
"llvm.vectorizer.";
5408 if (OldTag ==
"llvm.vectorizer.unroll")
5417 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5420 if (
T->getNumOperands() < 1)
5422 auto *OldTag = dyn_cast_or_null<MDString>(
T->getOperand(0));
5425 if (!OldTag->getString().starts_with(
"llvm.vectorizer."))
5432 for (
unsigned I = 1, E =
T->getNumOperands();
I != E; ++
I)
5439 auto *
T = dyn_cast<MDTuple>(&
N);
5458 if (((
T.isAMDGPU() && !
T.isAMDGCN()) ||
5459 (
T.isSPIR() || (
T.isSPIRV() && !
T.isSPIRVLogical()))) &&
5460 !
DL.contains(
"-G") && !
DL.starts_with(
"G")) {
5461 return DL.
empty() ? std::string(
"G1") : (
DL +
"-G1").str();
5464 if (
T.isLoongArch64() ||
T.isRISCV64()) {
5466 auto I =
DL.find(
"-n64-");
5468 return (
DL.take_front(
I) +
"-n32:64-" +
DL.drop_front(
I + 5)).str();
5472 std::string Res =
DL.str();
5476 if (!
DL.contains(
"-G") && !
DL.starts_with(
"G"))
5477 Res.append(Res.empty() ?
"G1" :
"-G1");
5482 if (!
DL.contains(
"-ni") && !
DL.starts_with(
"ni"))
5483 Res.append(
"-ni:7:8:9");
5485 if (
DL.ends_with(
"ni:7"))
5487 if (
DL.ends_with(
"ni:7:8"))
5492 if (!
DL.contains(
"-p7") && !
DL.starts_with(
"p7"))
5493 Res.append(
"-p7:160:256:256:32");
5494 if (!
DL.contains(
"-p8") && !
DL.starts_with(
"p8"))
5495 Res.append(
"-p8:128:128");
5496 if (!
DL.contains(
"-p9") && !
DL.starts_with(
"p9"))
5497 Res.append(
"-p9:192:256:256:32");
5502 auto AddPtr32Ptr64AddrSpaces = [&
DL, &Res]() {
5505 StringRef AddrSpaces{
"-p270:32:32-p271:32:32-p272:64:64"};
5506 if (!
DL.contains(AddrSpaces)) {
5508 Regex R(
"^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
5509 if (R.match(Res, &
Groups))
5515 if (
T.isAArch64()) {
5517 if (!
DL.
empty() && !
DL.contains(
"-Fn32"))
5518 Res.append(
"-Fn32");
5519 AddPtr32Ptr64AddrSpaces();
5523 if (
T.isSPARC() || (
T.isMIPS64() && !
DL.contains(
"m:m")) ||
T.isPPC64() ||
5527 std::string I64 =
"-i64:64";
5528 std::string I128 =
"-i128:128";
5530 size_t Pos = Res.find(I64);
5531 if (Pos !=
size_t(-1))
5532 Res.insert(Pos + I64.size(), I128);
5540 AddPtr32Ptr64AddrSpaces();
5548 if (!
T.isOSIAMCU()) {
5549 std::string I128 =
"-i128:128";
5552 Regex R(
"^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
5553 if (R.match(Res, &
Groups))
5561 if (
T.isWindowsMSVCEnvironment() && !
T.isArch64Bit()) {
5563 auto I =
Ref.find(
"-f80:32-");
5565 Res = (
Ref.take_front(
I) +
"-f80:128-" +
Ref.drop_front(
I + 8)).str();
5573 Attribute A =
B.getAttribute(
"no-frame-pointer-elim");
5576 FramePointer =
A.getValueAsString() ==
"true" ?
"all" :
"none";
5577 B.removeAttribute(
"no-frame-pointer-elim");
5579 if (
B.contains(
"no-frame-pointer-elim-non-leaf")) {
5581 if (FramePointer !=
"all")
5582 FramePointer =
"non-leaf";
5583 B.removeAttribute(
"no-frame-pointer-elim-non-leaf");
5585 if (!FramePointer.
empty())
5586 B.addAttribute(
"frame-pointer", FramePointer);
5588 A =
B.getAttribute(
"null-pointer-is-valid");
5591 bool NullPointerIsValid =
A.getValueAsString() ==
"true";
5592 B.removeAttribute(
"null-pointer-is-valid");
5593 if (NullPointerIsValid)
5594 B.addAttribute(Attribute::NullPointerIsValid);
5604 return OBD.
getTag() ==
"clang.arc.attachedcall" &&
AMDGPU address space definition.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Metadata * upgradeLoopArgument(Metadata *MD)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDType * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getNumElements(Type *Ty)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static ConstantAggregateZero * get(Type *Ty)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
LinkageTypes getLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Type * getFloatTy()
Fetch the type representing a 32-bit floating point value.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Type * getBFloatTy()
Fetch the type representing a 16-bit brain floating point value.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
RetTy visitCallBase(CallBase &I)
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
This is an important class for using LLVM in a threaded context.
SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVMContext & getContext() const
Tracking metadata reference owned by Metadata.
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
bool IsNewDbgInfoFormat
Is this Module using intrinsics to record the position of debugging information, or non-intrinsic rec...
void setOperand(unsigned I, MDNode *New)
MDNode * getOperand(unsigned i) const
unsigned getNumOperands() const
A container for an operand bundle being viewed as a set of values rather than a set of uses.
ArrayRef< InputTy > inputs() const
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
static ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
ArrayRef< int > getShuffleMask() const
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getBFloatTy(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
AttributeMask typeIncompatible(Type *Ty, AttributeSet AS, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
This is an optimization pass for GlobalISel generic memory operations.
void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.