24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/IR/CallSite.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/InlineAsm.h"
28 #include "llvm/IR/Intrinsics.h"
29 #include "llvm/IR/MDBuilder.h"
32 using namespace clang;
33 using namespace CodeGen;
49 if (FD->hasAttr<AsmLabelAttr>())
50 Name = getMangledName(D);
54 llvm::FunctionType *Ty =
55 cast<llvm::FunctionType>(getTypes().ConvertType(FD->
getType()));
57 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
63 QualType T, llvm::IntegerType *IntType) {
66 if (V->getType()->isPointerTy())
67 return CGF.
Builder.CreatePtrToInt(V, IntType);
69 assert(V->getType() == IntType);
77 if (ResultType->isPointerTy())
78 return CGF.
Builder.CreateIntToPtr(V, ResultType);
80 assert(V->getType() == ResultType);
87 llvm::AtomicRMWInst::BinOp
Kind,
96 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
98 llvm::IntegerType *IntType =
101 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
107 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
110 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
121 Address, llvm::PointerType::getUnqual(Val->getType()),
"cast");
137 llvm::AtomicRMWInst::BinOp
Kind,
146 llvm::AtomicRMWInst::BinOp
Kind,
148 Instruction::BinaryOps Op,
149 bool Invert =
false) {
157 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
159 llvm::IntegerType *IntType =
162 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
167 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
171 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
172 Result = CGF.
Builder.CreateBinOp(Op, Result, Args[1]);
174 Result = CGF.
Builder.CreateBinOp(llvm::Instruction::Xor, Result,
175 llvm::ConstantInt::get(IntType, -1));
195 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
197 llvm::IntegerType *IntType = llvm::IntegerType::get(
199 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
205 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
209 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
210 llvm::AtomicOrdering::SequentiallyConsistent);
213 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
225 unsigned IntrinsicID) {
229 return CGF.
Builder.CreateCall(F, Src0);
235 unsigned IntrinsicID) {
240 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
246 unsigned IntrinsicID) {
252 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
258 unsigned IntrinsicID) {
263 return CGF.
Builder.CreateCall(F, {Src0, Src1});
269 llvm::CallInst *Call = CGF.
Builder.CreateCall(F, V);
270 Call->setDoesNotAccessMemory();
280 int Width = Ty->getPrimitiveSizeInBits();
281 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
283 if (Ty->isPPC_FP128Ty()) {
293 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
294 V = CGF.
Builder.CreateLShr(V, ShiftCst);
298 IntTy = llvm::IntegerType::get(C, Width);
299 V = CGF.
Builder.CreateTrunc(V, IntTy);
301 Value *Zero = llvm::Constant::getNullValue(IntTy);
302 return CGF.
Builder.CreateICmpSLT(V, Zero);
325 assert(X->getType() == Y->getType() &&
326 "Arguments must be the same type. (Did you forget to make sure both "
327 "arguments have the same integer width?)");
331 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
332 return CGF.
Builder.CreateExtractValue(Tmp, 0);
336 unsigned IntrinsicID,
339 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
341 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
342 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
347 struct WidthAndSignedness {
353 static WidthAndSignedness
356 assert(Type->
isIntegerType() &&
"Given type is not an integer.");
359 return {Width, Signed};
365 static struct WidthAndSignedness
367 assert(Types.size() > 0 &&
"Empty list of types.");
371 for (
const auto &
Type : Types) {
372 Signed |=
Type.Signed;
380 for (
const auto &
Type : Types) {
381 unsigned MinWidth =
Type.Width + (Signed && !
Type.Signed);
382 if (Width < MinWidth) {
387 return {Width, Signed};
392 if (ArgValue->getType() != DestType)
394 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
396 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
397 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
406 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
411 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1,
true);
415 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
416 llvm::IntegerType *ResType) {
419 return emitBuiltinObjectSize(E, Type, ResType);
420 return ConstantInt::get(ResType, ObjectSize,
true);
429 CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
430 llvm::IntegerType *ResType) {
435 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
436 if (Param !=
nullptr && PS !=
nullptr &&
438 auto Iter = SizeArguments.find(Param);
439 assert(Iter != SizeArguments.end());
442 auto DIter = LocalDeclMap.find(D);
443 assert(DIter != LocalDeclMap.end());
445 return EmitLoadOfScalar(DIter->second,
false,
446 getContext().getSizeType(), E->getLocStart());
458 auto *CI = ConstantInt::get(
Builder.getInt1Ty(), (Type & 2) >> 1);
461 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
462 return Builder.CreateCall(F, {EmitScalarExpr(E), CI});
466 unsigned BuiltinID,
const CallExpr *E,
473 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
476 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
482 case Builtin::BI__builtin___CFStringMakeConstantString:
483 case Builtin::BI__builtin___NSStringMakeConstantString:
485 case Builtin::BI__builtin_stdarg_start:
486 case Builtin::BI__builtin_va_start:
487 case Builtin::BI__va_start:
488 case Builtin::BI__builtin_va_end:
490 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
491 ? EmitScalarExpr(E->
getArg(0))
492 : EmitVAListRef(E->
getArg(0)).getPointer(),
493 BuiltinID != Builtin::BI__builtin_va_end));
494 case Builtin::BI__builtin_va_copy: {
495 Value *DstPtr = EmitVAListRef(E->
getArg(0)).getPointer();
496 Value *SrcPtr = EmitVAListRef(E->
getArg(1)).getPointer();
500 DstPtr =
Builder.CreateBitCast(DstPtr, Type);
501 SrcPtr =
Builder.CreateBitCast(SrcPtr, Type);
505 case Builtin::BI__builtin_abs:
506 case Builtin::BI__builtin_labs:
507 case Builtin::BI__builtin_llabs: {
512 Builder.CreateICmpSGE(ArgValue,
513 llvm::Constant::getNullValue(ArgValue->getType()),
516 Builder.CreateSelect(CmpResult, ArgValue, NegOp,
"abs");
520 case Builtin::BI__builtin_fabs:
521 case Builtin::BI__builtin_fabsf:
522 case Builtin::BI__builtin_fabsl: {
525 case Builtin::BI__builtin_fmod:
526 case Builtin::BI__builtin_fmodf:
527 case Builtin::BI__builtin_fmodl: {
533 case Builtin::BI__builtin_copysign:
534 case Builtin::BI__builtin_copysignf:
535 case Builtin::BI__builtin_copysignl: {
538 case Builtin::BI__builtin_ceil:
539 case Builtin::BI__builtin_ceilf:
540 case Builtin::BI__builtin_ceill: {
543 case Builtin::BI__builtin_floor:
544 case Builtin::BI__builtin_floorf:
545 case Builtin::BI__builtin_floorl: {
548 case Builtin::BI__builtin_trunc:
549 case Builtin::BI__builtin_truncf:
550 case Builtin::BI__builtin_truncl: {
553 case Builtin::BI__builtin_rint:
554 case Builtin::BI__builtin_rintf:
555 case Builtin::BI__builtin_rintl: {
558 case Builtin::BI__builtin_nearbyint:
559 case Builtin::BI__builtin_nearbyintf:
560 case Builtin::BI__builtin_nearbyintl: {
563 case Builtin::BI__builtin_round:
564 case Builtin::BI__builtin_roundf:
565 case Builtin::BI__builtin_roundl: {
568 case Builtin::BI__builtin_fmin:
569 case Builtin::BI__builtin_fminf:
570 case Builtin::BI__builtin_fminl: {
573 case Builtin::BI__builtin_fmax:
574 case Builtin::BI__builtin_fmaxf:
575 case Builtin::BI__builtin_fmaxl: {
578 case Builtin::BI__builtin_conj:
579 case Builtin::BI__builtin_conjf:
580 case Builtin::BI__builtin_conjl: {
582 Value *Real = ComplexVal.first;
583 Value *Imag = ComplexVal.second;
585 Imag->getType()->isFPOrFPVectorTy()
586 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
587 : llvm::Constant::getNullValue(Imag->getType());
589 Imag =
Builder.CreateFSub(Zero, Imag,
"sub");
592 case Builtin::BI__builtin_creal:
593 case Builtin::BI__builtin_crealf:
594 case Builtin::BI__builtin_creall:
595 case Builtin::BIcreal:
596 case Builtin::BIcrealf:
597 case Builtin::BIcreall: {
602 case Builtin::BI__builtin_cimag:
603 case Builtin::BI__builtin_cimagf:
604 case Builtin::BI__builtin_cimagl:
605 case Builtin::BIcimag:
606 case Builtin::BIcimagf:
607 case Builtin::BIcimagl: {
612 case Builtin::BI__builtin_ctzs:
613 case Builtin::BI__builtin_ctz:
614 case Builtin::BI__builtin_ctzl:
615 case Builtin::BI__builtin_ctzll: {
619 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
622 Value *ZeroUndef =
Builder.getInt1(getTarget().isCLZForZeroUndef());
623 Value *Result =
Builder.CreateCall(F, {ArgValue, ZeroUndef});
624 if (Result->getType() != ResultType)
625 Result =
Builder.CreateIntCast(Result, ResultType,
true,
629 case Builtin::BI__builtin_clzs:
630 case Builtin::BI__builtin_clz:
631 case Builtin::BI__builtin_clzl:
632 case Builtin::BI__builtin_clzll: {
636 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
639 Value *ZeroUndef =
Builder.getInt1(getTarget().isCLZForZeroUndef());
640 Value *Result =
Builder.CreateCall(F, {ArgValue, ZeroUndef});
641 if (Result->getType() != ResultType)
642 Result =
Builder.CreateIntCast(Result, ResultType,
true,
646 case Builtin::BI__builtin_ffs:
647 case Builtin::BI__builtin_ffsl:
648 case Builtin::BI__builtin_ffsll: {
653 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
658 llvm::ConstantInt::get(ArgType, 1));
659 Value *Zero = llvm::Constant::getNullValue(ArgType);
660 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
661 Value *Result =
Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
662 if (Result->getType() != ResultType)
663 Result =
Builder.CreateIntCast(Result, ResultType,
true,
667 case Builtin::BI__builtin_parity:
668 case Builtin::BI__builtin_parityl:
669 case Builtin::BI__builtin_parityll: {
674 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
678 Value *Result =
Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
679 if (Result->getType() != ResultType)
680 Result =
Builder.CreateIntCast(Result, ResultType,
true,
684 case Builtin::BI__builtin_popcount:
685 case Builtin::BI__builtin_popcountl:
686 case Builtin::BI__builtin_popcountll: {
690 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
694 if (Result->getType() != ResultType)
695 Result =
Builder.CreateIntCast(Result, ResultType,
true,
699 case Builtin::BI__builtin_unpredictable: {
705 case Builtin::BI__builtin_expect: {
709 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
713 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
716 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
718 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
721 case Builtin::BI__builtin_assume_aligned: {
726 Value *AlignmentValue = EmitScalarExpr(E->
getArg(1));
727 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
728 unsigned Alignment = (
unsigned) AlignmentCI->getZExtValue();
730 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
733 case Builtin::BI__assume:
734 case Builtin::BI__builtin_assume: {
739 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
742 case Builtin::BI__builtin_bswap16:
743 case Builtin::BI__builtin_bswap32:
744 case Builtin::BI__builtin_bswap64: {
747 case Builtin::BI__builtin_bitreverse8:
748 case Builtin::BI__builtin_bitreverse16:
749 case Builtin::BI__builtin_bitreverse32:
750 case Builtin::BI__builtin_bitreverse64: {
753 case Builtin::BI__builtin_object_size: {
756 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->
getType()));
762 case Builtin::BI__builtin_prefetch: {
766 llvm::ConstantInt::get(Int32Ty, 0);
768 llvm::ConstantInt::get(Int32Ty, 3);
769 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
773 case Builtin::BI__builtin_readcyclecounter: {
774 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
777 case Builtin::BI__builtin___clear_cache: {
778 Value *Begin = EmitScalarExpr(E->getArg(0));
779 Value *
End = EmitScalarExpr(E->getArg(1));
780 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
783 case Builtin::BI__builtin_trap:
785 case Builtin::BI__debugbreak:
786 return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
787 case Builtin::BI__builtin_unreachable: {
788 if (SanOpts.has(SanitizerKind::Unreachable)) {
789 SanitizerScope SanScope(
this);
790 EmitCheck(std::make_pair(static_cast<llvm::Value *>(
Builder.getFalse()),
791 SanitizerKind::Unreachable),
792 "builtin_unreachable", EmitCheckSourceLocation(E->
getExprLoc()),
798 EmitBlock(createBasicBlock(
"unreachable.cont"));
803 case Builtin::BI__builtin_powi:
804 case Builtin::BI__builtin_powif:
805 case Builtin::BI__builtin_powil: {
806 Value *
Base = EmitScalarExpr(E->getArg(0));
807 Value *Exponent = EmitScalarExpr(E->getArg(1));
809 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
813 case Builtin::BI__builtin_isgreater:
814 case Builtin::BI__builtin_isgreaterequal:
815 case Builtin::BI__builtin_isless:
816 case Builtin::BI__builtin_islessequal:
817 case Builtin::BI__builtin_islessgreater:
818 case Builtin::BI__builtin_isunordered: {
821 Value *LHS = EmitScalarExpr(E->getArg(0));
822 Value *RHS = EmitScalarExpr(E->getArg(1));
825 default: llvm_unreachable(
"Unknown ordered comparison");
826 case Builtin::BI__builtin_isgreater:
827 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
829 case Builtin::BI__builtin_isgreaterequal:
830 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
832 case Builtin::BI__builtin_isless:
833 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
835 case Builtin::BI__builtin_islessequal:
836 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
838 case Builtin::BI__builtin_islessgreater:
839 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
841 case Builtin::BI__builtin_isunordered:
842 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
846 return RValue::get(
Builder.CreateZExt(LHS, ConvertType(E->
getType())));
848 case Builtin::BI__builtin_isnan: {
849 Value *V = EmitScalarExpr(E->getArg(0));
850 V =
Builder.CreateFCmpUNO(V, V,
"cmp");
851 return RValue::get(
Builder.CreateZExt(V, ConvertType(E->
getType())));
854 case Builtin::BI__builtin_isinf:
855 case Builtin::BI__builtin_isfinite: {
859 Value *V = EmitScalarExpr(E->getArg(0));
861 Constant *Infinity = ConstantFP::getInfinity(V->getType());
862 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
865 Value *FCmp =
Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
866 return RValue::get(
Builder.CreateZExt(FCmp, ConvertType(E->
getType())));
869 case Builtin::BI__builtin_isinf_sign: {
871 Value *Arg = EmitScalarExpr(E->getArg(0));
874 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
877 llvm::Type *IntTy = ConvertType(E->
getType());
878 Value *Zero = Constant::getNullValue(IntTy);
879 Value *One = ConstantInt::get(IntTy, 1);
880 Value *NegativeOne = ConstantInt::get(IntTy, -1);
881 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
883 return RValue::get(Result);
886 case Builtin::BI__builtin_isnormal: {
888 Value *V = EmitScalarExpr(E->getArg(0));
892 Value *IsLessThanInf =
893 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),
"isinf");
894 APFloat Smallest = APFloat::getSmallestNormalized(
895 getContext().getFloatTypeSemantics(E->getArg(0)->
getType()));
897 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
899 V =
Builder.CreateAnd(Eq, IsLessThanInf,
"and");
900 V =
Builder.CreateAnd(V, IsNormal,
"and");
901 return RValue::get(
Builder.CreateZExt(V, ConvertType(E->
getType())));
904 case Builtin::BI__builtin_fpclassify: {
905 Value *V = EmitScalarExpr(E->getArg(5));
906 llvm::Type *Ty = ConvertType(E->getArg(5)->
getType());
909 BasicBlock *Begin =
Builder.GetInsertBlock();
910 BasicBlock *End = createBasicBlock(
"fpclassify_end", this->CurFn);
914 "fpclassify_result");
918 Value *IsZero =
Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
920 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
921 BasicBlock *NotZero = createBasicBlock(
"fpclassify_not_zero", this->CurFn);
922 Builder.CreateCondBr(IsZero, End, NotZero);
923 Result->addIncoming(ZeroLiteral, Begin);
926 Builder.SetInsertPoint(NotZero);
928 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
929 BasicBlock *NotNan = createBasicBlock(
"fpclassify_not_nan", this->CurFn);
930 Builder.CreateCondBr(IsNan, End, NotNan);
931 Result->addIncoming(NanLiteral, NotZero);
934 Builder.SetInsertPoint(NotNan);
937 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
939 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
940 BasicBlock *NotInf = createBasicBlock(
"fpclassify_not_inf", this->CurFn);
941 Builder.CreateCondBr(IsInf, End, NotInf);
942 Result->addIncoming(InfLiteral, NotNan);
945 Builder.SetInsertPoint(NotInf);
946 APFloat Smallest = APFloat::getSmallestNormalized(
947 getContext().getFloatTypeSemantics(E->getArg(5)->
getType()));
949 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
951 Value *NormalResult =
952 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
953 EmitScalarExpr(E->getArg(3)));
955 Result->addIncoming(NormalResult, NotInf);
959 return RValue::get(Result);
962 case Builtin::BIalloca:
963 case Builtin::BI_alloca:
964 case Builtin::BI__builtin_alloca: {
965 Value *Size = EmitScalarExpr(E->getArg(0));
966 return RValue::get(
Builder.CreateAlloca(
Builder.getInt8Ty(), Size));
968 case Builtin::BIbzero:
969 case Builtin::BI__builtin_bzero: {
970 Address Dest = EmitPointerWithAlignment(E->getArg(0));
971 Value *SizeVal = EmitScalarExpr(E->getArg(1));
977 case Builtin::BImemcpy:
978 case Builtin::BI__builtin_memcpy: {
979 Address Dest = EmitPointerWithAlignment(E->getArg(0));
980 Address Src = EmitPointerWithAlignment(E->getArg(1));
981 Value *SizeVal = EmitScalarExpr(E->getArg(2));
984 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->
getType(),
986 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
990 case Builtin::BI__builtin___memcpy_chk: {
992 llvm::APSInt Size, DstSize;
996 if (Size.ugt(DstSize))
998 Address Dest = EmitPointerWithAlignment(E->getArg(0));
999 Address Src = EmitPointerWithAlignment(E->getArg(1));
1000 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1001 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
1005 case Builtin::BI__builtin_objc_memmove_collectable: {
1006 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
1007 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
1008 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1009 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*
this,
1010 DestAddr, SrcAddr, SizeVal);
1014 case Builtin::BI__builtin___memmove_chk: {
1016 llvm::APSInt Size, DstSize;
1020 if (Size.ugt(DstSize))
1022 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1023 Address Src = EmitPointerWithAlignment(E->getArg(1));
1024 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1025 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
1029 case Builtin::BImemmove:
1030 case Builtin::BI__builtin_memmove: {
1031 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1032 Address Src = EmitPointerWithAlignment(E->getArg(1));
1033 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1036 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->
getType(),
1038 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
1041 case Builtin::BImemset:
1042 case Builtin::BI__builtin_memset: {
1043 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1044 Value *ByteVal =
Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1046 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1049 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
1052 case Builtin::BI__builtin___memset_chk: {
1054 llvm::APSInt Size, DstSize;
1058 if (Size.ugt(DstSize))
1060 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1061 Value *ByteVal =
Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1063 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1064 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
1067 case Builtin::BI__builtin_dwarf_cfa: {
1078 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
1079 return RValue::get(
Builder.CreateCall(F,
1080 llvm::ConstantInt::get(Int32Ty, Offset)));
1082 case Builtin::BI__builtin_return_address: {
1084 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy,
this);
1085 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1086 return RValue::get(
Builder.CreateCall(F, Depth));
1088 case Builtin::BI__builtin_frame_address: {
1090 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy,
this);
1091 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
1092 return RValue::get(
Builder.CreateCall(F, Depth));
1094 case Builtin::BI__builtin_extract_return_addr: {
1096 Value *Result = getTargetHooks().decodeReturnAddress(*
this, Address);
1097 return RValue::get(Result);
1099 case Builtin::BI__builtin_frob_return_addr: {
1100 Value *Address = EmitScalarExpr(E->getArg(0));
1101 Value *Result = getTargetHooks().encodeReturnAddress(*
this, Address);
1102 return RValue::get(Result);
1104 case Builtin::BI__builtin_dwarf_sp_column: {
1105 llvm::IntegerType *Ty
1106 = cast<llvm::IntegerType>(ConvertType(E->
getType()));
1107 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
1109 CGM.ErrorUnsupported(E,
"__builtin_dwarf_sp_column");
1110 return RValue::get(llvm::UndefValue::get(Ty));
1112 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
1114 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
1115 Value *Address = EmitScalarExpr(E->getArg(0));
1116 if (getTargetHooks().initDwarfEHRegSizeTable(*
this, Address))
1117 CGM.ErrorUnsupported(E,
"__builtin_init_dwarf_reg_size_table");
1118 return RValue::get(llvm::UndefValue::get(ConvertType(E->
getType())));
1120 case Builtin::BI__builtin_eh_return: {
1121 Value *Int = EmitScalarExpr(E->getArg(0));
1122 Value *Ptr = EmitScalarExpr(E->getArg(1));
1124 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
1125 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
1126 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
1127 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
1128 ? Intrinsic::eh_return_i32
1129 : Intrinsic::eh_return_i64);
1130 Builder.CreateCall(F, {Int, Ptr});
1134 EmitBlock(createBasicBlock(
"builtin_eh_return.cont"));
1136 return RValue::get(
nullptr);
1138 case Builtin::BI__builtin_unwind_init: {
1139 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
1140 return RValue::get(
Builder.CreateCall(F));
1142 case Builtin::BI__builtin_extend_pointer: {
1154 Value *Ptr = EmitScalarExpr(E->getArg(0));
1155 Value *Result =
Builder.CreatePtrToInt(Ptr, IntPtrTy,
"extend.cast");
1158 if (IntPtrTy->getBitWidth() == 64)
1159 return RValue::get(Result);
1162 if (getTargetHooks().extendPointerWithSExt())
1163 return RValue::get(
Builder.CreateSExt(Result, Int64Ty,
"extend.sext"));
1165 return RValue::get(
Builder.CreateZExt(Result, Int64Ty,
"extend.zext"));
1167 case Builtin::BI__builtin_setjmp: {
1169 Address Buf = EmitPointerWithAlignment(E->getArg(0));
1173 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
1174 ConstantInt::get(Int32Ty, 0));
1175 Builder.CreateStore(FrameAddr, Buf);
1179 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
1180 Address StackSaveSlot =
1181 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
1182 Builder.CreateStore(StackAddr, StackSaveSlot);
1185 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
1186 Buf =
Builder.CreateBitCast(Buf, Int8PtrTy);
1189 case Builtin::BI__builtin_longjmp: {
1190 Value *Buf = EmitScalarExpr(E->getArg(0));
1191 Buf =
Builder.CreateBitCast(Buf, Int8PtrTy);
1194 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
1200 EmitBlock(createBasicBlock(
"longjmp.cont"));
1202 return RValue::get(
nullptr);
1204 case Builtin::BI__sync_fetch_and_add:
1205 case Builtin::BI__sync_fetch_and_sub:
1206 case Builtin::BI__sync_fetch_and_or:
1207 case Builtin::BI__sync_fetch_and_and:
1208 case Builtin::BI__sync_fetch_and_xor:
1209 case Builtin::BI__sync_fetch_and_nand:
1210 case Builtin::BI__sync_add_and_fetch:
1211 case Builtin::BI__sync_sub_and_fetch:
1212 case Builtin::BI__sync_and_and_fetch:
1213 case Builtin::BI__sync_or_and_fetch:
1214 case Builtin::BI__sync_xor_and_fetch:
1215 case Builtin::BI__sync_nand_and_fetch:
1216 case Builtin::BI__sync_val_compare_and_swap:
1217 case Builtin::BI__sync_bool_compare_and_swap:
1218 case Builtin::BI__sync_lock_test_and_set:
1219 case Builtin::BI__sync_lock_release:
1220 case Builtin::BI__sync_swap:
1221 llvm_unreachable(
"Shouldn't make it through sema");
1222 case Builtin::BI__sync_fetch_and_add_1:
1223 case Builtin::BI__sync_fetch_and_add_2:
1224 case Builtin::BI__sync_fetch_and_add_4:
1225 case Builtin::BI__sync_fetch_and_add_8:
1226 case Builtin::BI__sync_fetch_and_add_16:
1228 case Builtin::BI__sync_fetch_and_sub_1:
1229 case Builtin::BI__sync_fetch_and_sub_2:
1230 case Builtin::BI__sync_fetch_and_sub_4:
1231 case Builtin::BI__sync_fetch_and_sub_8:
1232 case Builtin::BI__sync_fetch_and_sub_16:
1234 case Builtin::BI__sync_fetch_and_or_1:
1235 case Builtin::BI__sync_fetch_and_or_2:
1236 case Builtin::BI__sync_fetch_and_or_4:
1237 case Builtin::BI__sync_fetch_and_or_8:
1238 case Builtin::BI__sync_fetch_and_or_16:
1240 case Builtin::BI__sync_fetch_and_and_1:
1241 case Builtin::BI__sync_fetch_and_and_2:
1242 case Builtin::BI__sync_fetch_and_and_4:
1243 case Builtin::BI__sync_fetch_and_and_8:
1244 case Builtin::BI__sync_fetch_and_and_16:
1246 case Builtin::BI__sync_fetch_and_xor_1:
1247 case Builtin::BI__sync_fetch_and_xor_2:
1248 case Builtin::BI__sync_fetch_and_xor_4:
1249 case Builtin::BI__sync_fetch_and_xor_8:
1250 case Builtin::BI__sync_fetch_and_xor_16:
1252 case Builtin::BI__sync_fetch_and_nand_1:
1253 case Builtin::BI__sync_fetch_and_nand_2:
1254 case Builtin::BI__sync_fetch_and_nand_4:
1255 case Builtin::BI__sync_fetch_and_nand_8:
1256 case Builtin::BI__sync_fetch_and_nand_16:
1260 case Builtin::BI__sync_fetch_and_min:
1262 case Builtin::BI__sync_fetch_and_max:
1264 case Builtin::BI__sync_fetch_and_umin:
1266 case Builtin::BI__sync_fetch_and_umax:
1269 case Builtin::BI__sync_add_and_fetch_1:
1270 case Builtin::BI__sync_add_and_fetch_2:
1271 case Builtin::BI__sync_add_and_fetch_4:
1272 case Builtin::BI__sync_add_and_fetch_8:
1273 case Builtin::BI__sync_add_and_fetch_16:
1275 llvm::Instruction::Add);
1276 case Builtin::BI__sync_sub_and_fetch_1:
1277 case Builtin::BI__sync_sub_and_fetch_2:
1278 case Builtin::BI__sync_sub_and_fetch_4:
1279 case Builtin::BI__sync_sub_and_fetch_8:
1280 case Builtin::BI__sync_sub_and_fetch_16:
1282 llvm::Instruction::Sub);
1283 case Builtin::BI__sync_and_and_fetch_1:
1284 case Builtin::BI__sync_and_and_fetch_2:
1285 case Builtin::BI__sync_and_and_fetch_4:
1286 case Builtin::BI__sync_and_and_fetch_8:
1287 case Builtin::BI__sync_and_and_fetch_16:
1290 case Builtin::BI__sync_or_and_fetch_1:
1291 case Builtin::BI__sync_or_and_fetch_2:
1292 case Builtin::BI__sync_or_and_fetch_4:
1293 case Builtin::BI__sync_or_and_fetch_8:
1294 case Builtin::BI__sync_or_and_fetch_16:
1296 llvm::Instruction::Or);
1297 case Builtin::BI__sync_xor_and_fetch_1:
1298 case Builtin::BI__sync_xor_and_fetch_2:
1299 case Builtin::BI__sync_xor_and_fetch_4:
1300 case Builtin::BI__sync_xor_and_fetch_8:
1301 case Builtin::BI__sync_xor_and_fetch_16:
1303 llvm::Instruction::Xor);
1304 case Builtin::BI__sync_nand_and_fetch_1:
1305 case Builtin::BI__sync_nand_and_fetch_2:
1306 case Builtin::BI__sync_nand_and_fetch_4:
1307 case Builtin::BI__sync_nand_and_fetch_8:
1308 case Builtin::BI__sync_nand_and_fetch_16:
1312 case Builtin::BI__sync_val_compare_and_swap_1:
1313 case Builtin::BI__sync_val_compare_and_swap_2:
1314 case Builtin::BI__sync_val_compare_and_swap_4:
1315 case Builtin::BI__sync_val_compare_and_swap_8:
1316 case Builtin::BI__sync_val_compare_and_swap_16:
1319 case Builtin::BI__sync_bool_compare_and_swap_1:
1320 case Builtin::BI__sync_bool_compare_and_swap_2:
1321 case Builtin::BI__sync_bool_compare_and_swap_4:
1322 case Builtin::BI__sync_bool_compare_and_swap_8:
1323 case Builtin::BI__sync_bool_compare_and_swap_16:
1326 case Builtin::BI__sync_swap_1:
1327 case Builtin::BI__sync_swap_2:
1328 case Builtin::BI__sync_swap_4:
1329 case Builtin::BI__sync_swap_8:
1330 case Builtin::BI__sync_swap_16:
1333 case Builtin::BI__sync_lock_test_and_set_1:
1334 case Builtin::BI__sync_lock_test_and_set_2:
1335 case Builtin::BI__sync_lock_test_and_set_4:
1336 case Builtin::BI__sync_lock_test_and_set_8:
1337 case Builtin::BI__sync_lock_test_and_set_16:
1340 case Builtin::BI__sync_lock_release_1:
1341 case Builtin::BI__sync_lock_release_2:
1342 case Builtin::BI__sync_lock_release_4:
1343 case Builtin::BI__sync_lock_release_8:
1344 case Builtin::BI__sync_lock_release_16: {
1345 Value *Ptr = EmitScalarExpr(E->getArg(0));
1347 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1348 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1350 Ptr =
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1351 llvm::StoreInst *
Store =
1352 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
1354 Store->setAtomic(llvm::AtomicOrdering::Release);
1355 return RValue::get(
nullptr);
1358 case Builtin::BI__sync_synchronize: {
1366 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
1367 return RValue::get(
nullptr);
1370 case Builtin::BI__builtin_nontemporal_load:
1372 case Builtin::BI__builtin_nontemporal_store:
1374 case Builtin::BI__c11_atomic_is_lock_free:
1375 case Builtin::BI__atomic_is_lock_free: {
1379 const char *LibCallName =
"__atomic_is_lock_free";
1381 Args.
add(RValue::get(EmitScalarExpr(E->getArg(0))),
1382 getContext().getSizeType());
1383 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1384 Args.
add(RValue::get(EmitScalarExpr(E->getArg(1))),
1385 getContext().VoidPtrTy);
1387 Args.
add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1388 getContext().VoidPtrTy);
1390 CGM.getTypes().arrangeBuiltinFunctionCall(E->
getType(), Args);
1391 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1392 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1396 case Builtin::BI__atomic_test_and_set: {
1403 Value *Ptr = EmitScalarExpr(E->getArg(0));
1404 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1405 Ptr =
Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1407 Value *Order = EmitScalarExpr(E->getArg(1));
1408 if (isa<llvm::ConstantInt>(Order)) {
1409 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1410 AtomicRMWInst *Result =
nullptr;
1414 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1415 llvm::AtomicOrdering::Monotonic);
1419 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1420 llvm::AtomicOrdering::Acquire);
1423 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1424 llvm::AtomicOrdering::Release);
1428 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1429 llvm::AtomicOrdering::AcquireRelease);
1432 Result =
Builder.CreateAtomicRMW(
1433 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1434 llvm::AtomicOrdering::SequentiallyConsistent);
1437 Result->setVolatile(Volatile);
1438 return RValue::get(
Builder.CreateIsNotNull(Result,
"tobool"));
1441 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1443 llvm::BasicBlock *BBs[5] = {
1444 createBasicBlock(
"monotonic", CurFn),
1445 createBasicBlock(
"acquire", CurFn),
1446 createBasicBlock(
"release", CurFn),
1447 createBasicBlock(
"acqrel", CurFn),
1448 createBasicBlock(
"seqcst", CurFn)
1450 llvm::AtomicOrdering Orders[5] = {
1451 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
1452 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
1453 llvm::AtomicOrdering::SequentiallyConsistent};
1455 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1456 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
1458 Builder.SetInsertPoint(ContBB);
1459 PHINode *Result =
Builder.CreatePHI(Int8Ty, 5,
"was_set");
1461 for (
unsigned i = 0; i < 5; ++i) {
1462 Builder.SetInsertPoint(BBs[i]);
1463 AtomicRMWInst *RMW =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1464 Ptr, NewVal, Orders[i]);
1465 RMW->setVolatile(Volatile);
1466 Result->addIncoming(RMW, BBs[i]);
1470 SI->addCase(
Builder.getInt32(0), BBs[0]);
1471 SI->addCase(
Builder.getInt32(1), BBs[1]);
1472 SI->addCase(
Builder.getInt32(2), BBs[1]);
1473 SI->addCase(
Builder.getInt32(3), BBs[2]);
1474 SI->addCase(
Builder.getInt32(4), BBs[3]);
1475 SI->addCase(
Builder.getInt32(5), BBs[4]);
1477 Builder.SetInsertPoint(ContBB);
1478 return RValue::get(
Builder.CreateIsNotNull(Result,
"tobool"));
1481 case Builtin::BI__atomic_clear: {
1486 Address Ptr = EmitPointerWithAlignment(E->getArg(0));
1487 unsigned AddrSpace = Ptr.
getPointer()->getType()->getPointerAddressSpace();
1488 Ptr =
Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1490 Value *Order = EmitScalarExpr(E->getArg(1));
1491 if (isa<llvm::ConstantInt>(Order)) {
1492 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1493 StoreInst *Store =
Builder.CreateStore(NewVal, Ptr, Volatile);
1497 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
1500 Store->setOrdering(llvm::AtomicOrdering::Release);
1503 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
1506 return RValue::get(
nullptr);
1509 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1511 llvm::BasicBlock *BBs[3] = {
1512 createBasicBlock(
"monotonic", CurFn),
1513 createBasicBlock(
"release", CurFn),
1514 createBasicBlock(
"seqcst", CurFn)
1516 llvm::AtomicOrdering Orders[3] = {
1517 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
1518 llvm::AtomicOrdering::SequentiallyConsistent};
1520 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1521 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
1523 for (
unsigned i = 0; i < 3; ++i) {
1524 Builder.SetInsertPoint(BBs[i]);
1525 StoreInst *Store =
Builder.CreateStore(NewVal, Ptr, Volatile);
1526 Store->setOrdering(Orders[i]);
1530 SI->addCase(
Builder.getInt32(0), BBs[0]);
1531 SI->addCase(
Builder.getInt32(3), BBs[1]);
1532 SI->addCase(
Builder.getInt32(5), BBs[2]);
1534 Builder.SetInsertPoint(ContBB);
1535 return RValue::get(
nullptr);
1538 case Builtin::BI__atomic_thread_fence:
1539 case Builtin::BI__atomic_signal_fence:
1540 case Builtin::BI__c11_atomic_thread_fence:
1541 case Builtin::BI__c11_atomic_signal_fence: {
1542 llvm::SynchronizationScope
Scope;
1543 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1544 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1545 Scope = llvm::SingleThread;
1547 Scope = llvm::CrossThread;
1548 Value *Order = EmitScalarExpr(E->getArg(0));
1549 if (isa<llvm::ConstantInt>(Order)) {
1550 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1557 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope);
1560 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope);
1563 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope);
1566 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
1570 return RValue::get(
nullptr);
1573 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1574 AcquireBB = createBasicBlock(
"acquire", CurFn);
1575 ReleaseBB = createBasicBlock(
"release", CurFn);
1576 AcqRelBB = createBasicBlock(
"acqrel", CurFn);
1577 SeqCstBB = createBasicBlock(
"seqcst", CurFn);
1578 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1580 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1581 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
1583 Builder.SetInsertPoint(AcquireBB);
1584 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope);
1586 SI->addCase(
Builder.getInt32(1), AcquireBB);
1587 SI->addCase(
Builder.getInt32(2), AcquireBB);
1589 Builder.SetInsertPoint(ReleaseBB);
1590 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope);
1592 SI->addCase(
Builder.getInt32(3), ReleaseBB);
1594 Builder.SetInsertPoint(AcqRelBB);
1595 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope);
1597 SI->addCase(
Builder.getInt32(4), AcqRelBB);
1599 Builder.SetInsertPoint(SeqCstBB);
1600 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1602 SI->addCase(
Builder.getInt32(5), SeqCstBB);
1604 Builder.SetInsertPoint(ContBB);
1605 return RValue::get(
nullptr);
1609 case Builtin::BIsqrt:
1610 case Builtin::BIsqrtf:
1611 case Builtin::BIsqrtl: {
1616 if (!FD->hasAttr<ConstAttr>())
1618 if (!(CGM.getCodeGenOpts().UnsafeFPMath ||
1619 CGM.getCodeGenOpts().NoNaNsFPMath))
1621 Value *Arg0 = EmitScalarExpr(E->getArg(0));
1622 llvm::Type *ArgType = Arg0->getType();
1624 return RValue::get(
Builder.CreateCall(F, Arg0));
1627 case Builtin::BI__builtin_pow:
1628 case Builtin::BI__builtin_powf:
1629 case Builtin::BI__builtin_powl:
1630 case Builtin::BIpow:
1631 case Builtin::BIpowf:
1632 case Builtin::BIpowl: {
1634 if (!FD->hasAttr<ConstAttr>())
1636 Value *Base = EmitScalarExpr(E->getArg(0));
1637 Value *Exponent = EmitScalarExpr(E->getArg(1));
1638 llvm::Type *ArgType = Base->getType();
1640 return RValue::get(
Builder.CreateCall(F, {Base, Exponent}));
1643 case Builtin::BIfma:
1644 case Builtin::BIfmaf:
1645 case Builtin::BIfmal:
1646 case Builtin::BI__builtin_fma:
1647 case Builtin::BI__builtin_fmaf:
1648 case Builtin::BI__builtin_fmal: {
1650 Value *FirstArg = EmitScalarExpr(E->getArg(0));
1651 llvm::Type *ArgType = FirstArg->getType();
1654 Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)),
1655 EmitScalarExpr(E->getArg(2))}));
1658 case Builtin::BI__builtin_signbit:
1659 case Builtin::BI__builtin_signbitf:
1660 case Builtin::BI__builtin_signbitl: {
1665 case Builtin::BI__builtin_annotation: {
1666 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1667 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1673 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1674 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->
getExprLoc()));
1676 case Builtin::BI__builtin_addcb:
1677 case Builtin::BI__builtin_addcs:
1678 case Builtin::BI__builtin_addc:
1679 case Builtin::BI__builtin_addcl:
1680 case Builtin::BI__builtin_addcll:
1681 case Builtin::BI__builtin_subcb:
1682 case Builtin::BI__builtin_subcs:
1683 case Builtin::BI__builtin_subc:
1684 case Builtin::BI__builtin_subcl:
1685 case Builtin::BI__builtin_subcll: {
1707 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
1708 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
1712 switch (BuiltinID) {
1713 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
1714 case Builtin::BI__builtin_addcb:
1715 case Builtin::BI__builtin_addcs:
1716 case Builtin::BI__builtin_addc:
1717 case Builtin::BI__builtin_addcl:
1718 case Builtin::BI__builtin_addcll:
1719 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1721 case Builtin::BI__builtin_subcb:
1722 case Builtin::BI__builtin_subcs:
1723 case Builtin::BI__builtin_subc:
1724 case Builtin::BI__builtin_subcl:
1725 case Builtin::BI__builtin_subcll:
1726 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1736 Sum1, Carryin, Carry2);
1739 Builder.CreateStore(CarryOut, CarryOutPtr);
1740 return RValue::get(Sum2);
1743 case Builtin::BI__builtin_add_overflow:
1744 case Builtin::BI__builtin_sub_overflow:
1745 case Builtin::BI__builtin_mul_overflow: {
1753 WidthAndSignedness LeftInfo =
1755 WidthAndSignedness RightInfo =
1757 WidthAndSignedness ResultInfo =
1759 WidthAndSignedness EncompassingInfo =
1762 llvm::Type *EncompassingLLVMTy =
1763 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
1765 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
1768 switch (BuiltinID) {
1770 llvm_unreachable(
"Unknown overflow builtin id.");
1771 case Builtin::BI__builtin_add_overflow:
1772 IntrinsicId = EncompassingInfo.Signed
1773 ? llvm::Intrinsic::sadd_with_overflow
1774 : llvm::Intrinsic::uadd_with_overflow;
1776 case Builtin::BI__builtin_sub_overflow:
1777 IntrinsicId = EncompassingInfo.Signed
1778 ? llvm::Intrinsic::ssub_with_overflow
1779 : llvm::Intrinsic::usub_with_overflow;
1781 case Builtin::BI__builtin_mul_overflow:
1782 IntrinsicId = EncompassingInfo.Signed
1783 ? llvm::Intrinsic::smul_with_overflow
1784 : llvm::Intrinsic::umul_with_overflow;
1790 Address ResultPtr = EmitPointerWithAlignment(ResultArg);
1793 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
1794 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
1800 if (EncompassingInfo.Width > ResultInfo.Width) {
1808 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
1810 Builder.CreateICmpNE(Result, ResultTruncExt);
1812 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
1813 Result = ResultTrunc;
1819 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
1821 return RValue::get(Overflow);
1824 case Builtin::BI__builtin_uadd_overflow:
1825 case Builtin::BI__builtin_uaddl_overflow:
1826 case Builtin::BI__builtin_uaddll_overflow:
1827 case Builtin::BI__builtin_usub_overflow:
1828 case Builtin::BI__builtin_usubl_overflow:
1829 case Builtin::BI__builtin_usubll_overflow:
1830 case Builtin::BI__builtin_umul_overflow:
1831 case Builtin::BI__builtin_umull_overflow:
1832 case Builtin::BI__builtin_umulll_overflow:
1833 case Builtin::BI__builtin_sadd_overflow:
1834 case Builtin::BI__builtin_saddl_overflow:
1835 case Builtin::BI__builtin_saddll_overflow:
1836 case Builtin::BI__builtin_ssub_overflow:
1837 case Builtin::BI__builtin_ssubl_overflow:
1838 case Builtin::BI__builtin_ssubll_overflow:
1839 case Builtin::BI__builtin_smul_overflow:
1840 case Builtin::BI__builtin_smull_overflow:
1841 case Builtin::BI__builtin_smulll_overflow: {
1848 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
1852 switch (BuiltinID) {
1853 default: llvm_unreachable(
"Unknown overflow builtin id.");
1854 case Builtin::BI__builtin_uadd_overflow:
1855 case Builtin::BI__builtin_uaddl_overflow:
1856 case Builtin::BI__builtin_uaddll_overflow:
1857 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1859 case Builtin::BI__builtin_usub_overflow:
1860 case Builtin::BI__builtin_usubl_overflow:
1861 case Builtin::BI__builtin_usubll_overflow:
1862 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1864 case Builtin::BI__builtin_umul_overflow:
1865 case Builtin::BI__builtin_umull_overflow:
1866 case Builtin::BI__builtin_umulll_overflow:
1867 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
1869 case Builtin::BI__builtin_sadd_overflow:
1870 case Builtin::BI__builtin_saddl_overflow:
1871 case Builtin::BI__builtin_saddll_overflow:
1872 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
1874 case Builtin::BI__builtin_ssub_overflow:
1875 case Builtin::BI__builtin_ssubl_overflow:
1876 case Builtin::BI__builtin_ssubll_overflow:
1877 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
1879 case Builtin::BI__builtin_smul_overflow:
1880 case Builtin::BI__builtin_smull_overflow:
1881 case Builtin::BI__builtin_smulll_overflow:
1882 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
1889 Builder.CreateStore(Sum, SumOutPtr);
1891 return RValue::get(Carry);
1893 case Builtin::BI__builtin_addressof:
1894 return RValue::get(EmitLValue(E->getArg(0)).getPointer());
1895 case Builtin::BI__builtin_operator_new:
1897 E->getArg(0),
false);
1898 case Builtin::BI__builtin_operator_delete:
1900 E->getArg(0),
true);
1901 case Builtin::BI__noop:
1903 return RValue::get(ConstantInt::get(IntTy, 0));
1904 case Builtin::BI__builtin_call_with_static_chain: {
1906 const Expr *Chain = E->getArg(1);
1908 EmitScalarExpr(Call->
getCallee()), Call, ReturnValue,
1911 case Builtin::BI_InterlockedExchange:
1912 case Builtin::BI_InterlockedExchangePointer:
1914 case Builtin::BI_InterlockedCompareExchangePointer: {
1916 llvm::IntegerType *IntType =
1917 IntegerType::get(getLLVMContext(),
1918 getContext().getTypeSize(E->
getType()));
1919 llvm::Type *IntPtrType = IntType->getPointerTo();
1922 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
1924 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
1925 RTy = Exchange->getType();
1926 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
1929 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
1932 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
1933 AtomicOrdering::SequentiallyConsistent,
1934 AtomicOrdering::SequentiallyConsistent);
1935 Result->setVolatile(
true);
1937 return RValue::get(
Builder.CreateIntToPtr(
Builder.CreateExtractValue(Result,
1941 case Builtin::BI_InterlockedCompareExchange: {
1942 AtomicCmpXchgInst *CXI =
Builder.CreateAtomicCmpXchg(
1943 EmitScalarExpr(E->getArg(0)),
1944 EmitScalarExpr(E->getArg(2)),
1945 EmitScalarExpr(E->getArg(1)),
1946 AtomicOrdering::SequentiallyConsistent,
1947 AtomicOrdering::SequentiallyConsistent);
1948 CXI->setVolatile(
true);
1949 return RValue::get(
Builder.CreateExtractValue(CXI, 0));
1951 case Builtin::BI_InterlockedIncrement: {
1952 llvm::Type *IntTy = ConvertType(E->
getType());
1953 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
1955 EmitScalarExpr(E->getArg(0)),
1956 ConstantInt::get(IntTy, 1),
1957 llvm::AtomicOrdering::SequentiallyConsistent);
1958 RMWI->setVolatile(
true);
1959 return RValue::get(
Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1)));
1961 case Builtin::BI_InterlockedDecrement: {
1962 llvm::Type *IntTy = ConvertType(E->
getType());
1963 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
1965 EmitScalarExpr(E->getArg(0)),
1966 ConstantInt::get(IntTy, 1),
1967 llvm::AtomicOrdering::SequentiallyConsistent);
1968 RMWI->setVolatile(
true);
1969 return RValue::get(
Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1)));
1971 case Builtin::BI_InterlockedExchangeAdd: {
1972 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
1974 EmitScalarExpr(E->getArg(0)),
1975 EmitScalarExpr(E->getArg(1)),
1976 llvm::AtomicOrdering::SequentiallyConsistent);
1977 RMWI->setVolatile(
true);
1978 return RValue::get(RMWI);
1980 case Builtin::BI__readfsdword: {
1981 llvm::Type *IntTy = ConvertType(E->
getType());
1983 Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)),
1984 llvm::PointerType::get(IntTy, 257));
1986 Builder.CreateDefaultAlignedLoad(IntToPtr,
true);
1987 return RValue::get(Load);
1990 case Builtin::BI__exception_code:
1991 case Builtin::BI_exception_code:
1992 return RValue::get(EmitSEHExceptionCode());
1993 case Builtin::BI__exception_info:
1994 case Builtin::BI_exception_info:
1995 return RValue::get(EmitSEHExceptionInfo());
1996 case Builtin::BI__abnormal_termination:
1997 case Builtin::BI_abnormal_termination:
1998 return RValue::get(EmitSEHAbnormalTermination());
1999 case Builtin::BI_setjmpex: {
2000 if (getTarget().getTriple().isOSMSVCRT()) {
2001 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2002 llvm::AttributeSet ReturnsTwiceAttr =
2003 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
2004 llvm::Attribute::ReturnsTwice);
2005 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
2006 llvm::FunctionType::get(IntTy, ArgTypes,
false),
2007 "_setjmpex", ReturnsTwiceAttr);
2009 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2011 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2012 ConstantInt::get(Int32Ty, 0));
2014 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
2015 CS.setAttributes(ReturnsTwiceAttr);
2016 return RValue::get(CS.getInstruction());
2020 case Builtin::BI_setjmp: {
2021 if (getTarget().getTriple().isOSMSVCRT()) {
2022 llvm::AttributeSet ReturnsTwiceAttr =
2023 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
2024 llvm::Attribute::ReturnsTwice);
2026 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2028 if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
2029 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
2030 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
2031 llvm::FunctionType::get(IntTy, ArgTypes,
true),
2032 "_setjmp3", ReturnsTwiceAttr);
2035 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
2037 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2038 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
2039 llvm::FunctionType::get(IntTy, ArgTypes,
false),
2040 "_setjmp", ReturnsTwiceAttr);
2042 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2043 ConstantInt::get(Int32Ty, 0));
2045 CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
2047 CS.setAttributes(ReturnsTwiceAttr);
2048 return RValue::get(CS.getInstruction());
2053 case Builtin::BI__GetExceptionInfo: {
2054 if (llvm::GlobalVariable *GV =
2055 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
2056 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
2061 case Builtin::BIread_pipe:
2062 case Builtin::BIwrite_pipe: {
2063 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2064 *Arg1 = EmitScalarExpr(E->getArg(1));
2067 unsigned GenericAS =
2069 llvm::Type *I8PTy = llvm::PointerType::get(
2070 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
2073 if (2U == E->getNumArgs()) {
2074 const char *
Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
2078 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy};
2079 llvm::FunctionType *FTy = llvm::FunctionType::get(
2082 return RValue::get(
Builder.CreateCall(
2083 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast}));
2085 assert(4 == E->getNumArgs() &&
2086 "Illegal number of parameters to pipe function");
2087 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
2090 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy};
2091 Value *Arg2 = EmitScalarExpr(E->getArg(2)),
2092 *Arg3 = EmitScalarExpr(E->getArg(3));
2093 llvm::FunctionType *FTy = llvm::FunctionType::get(
2098 if (Arg2->getType() != Int32Ty)
2099 Arg2 =
Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
2100 return RValue::get(
Builder.CreateCall(
2101 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast}));
2106 case Builtin::BIreserve_read_pipe:
2107 case Builtin::BIreserve_write_pipe:
2108 case Builtin::BIwork_group_reserve_read_pipe:
2109 case Builtin::BIwork_group_reserve_write_pipe:
2110 case Builtin::BIsub_group_reserve_read_pipe:
2111 case Builtin::BIsub_group_reserve_write_pipe: {
2114 if (BuiltinID == Builtin::BIreserve_read_pipe)
2115 Name =
"__reserve_read_pipe";
2116 else if (BuiltinID == Builtin::BIreserve_write_pipe)
2117 Name =
"__reserve_write_pipe";
2118 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
2119 Name =
"__work_group_reserve_read_pipe";
2120 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
2121 Name =
"__work_group_reserve_write_pipe";
2122 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
2123 Name =
"__sub_group_reserve_read_pipe";
2125 Name =
"__sub_group_reserve_write_pipe";
2127 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2128 *Arg1 = EmitScalarExpr(E->getArg(1));
2129 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
2132 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty};
2133 llvm::FunctionType *FTy = llvm::FunctionType::get(
2137 if (Arg1->getType() != Int32Ty)
2138 Arg1 =
Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
2140 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1}));
2144 case Builtin::BIcommit_read_pipe:
2145 case Builtin::BIcommit_write_pipe:
2146 case Builtin::BIwork_group_commit_read_pipe:
2147 case Builtin::BIwork_group_commit_write_pipe:
2148 case Builtin::BIsub_group_commit_read_pipe:
2149 case Builtin::BIsub_group_commit_write_pipe: {
2151 if (BuiltinID == Builtin::BIcommit_read_pipe)
2152 Name =
"__commit_read_pipe";
2153 else if (BuiltinID == Builtin::BIcommit_write_pipe)
2154 Name =
"__commit_write_pipe";
2155 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
2156 Name =
"__work_group_commit_read_pipe";
2157 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
2158 Name =
"__work_group_commit_write_pipe";
2159 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
2160 Name =
"__sub_group_commit_read_pipe";
2162 Name =
"__sub_group_commit_write_pipe";
2164 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2165 *Arg1 = EmitScalarExpr(E->getArg(1));
2168 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()};
2169 llvm::FunctionType *FTy =
2170 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
2174 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1}));
2177 case Builtin::BIget_pipe_num_packets:
2178 case Builtin::BIget_pipe_max_packets: {
2180 if (BuiltinID == Builtin::BIget_pipe_num_packets)
2181 Name =
"__get_pipe_num_packets";
2183 Name =
"__get_pipe_max_packets";
2186 Value *Arg0 = EmitScalarExpr(E->getArg(0));
2187 llvm::Type *ArgTys[] = {Arg0->getType()};
2188 llvm::FunctionType *FTy = llvm::FunctionType::get(
2192 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0}));
2196 case Builtin::BIto_global:
2197 case Builtin::BIto_local:
2198 case Builtin::BIto_private: {
2199 auto Arg0 = EmitScalarExpr(E->getArg(0));
2200 auto NewArgT = llvm::PointerType::get(Int8Ty,
2202 auto NewRetT = llvm::PointerType::get(Int8Ty,
2203 CGM.getContext().getTargetAddressSpace(
2205 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
2207 if (Arg0->getType()->getPointerAddressSpace() !=
2208 NewArgT->getPointerAddressSpace())
2209 NewArg =
Builder.CreateAddrSpaceCast(Arg0, NewArgT);
2211 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
2212 auto NewName = std::string(
"__") + E->getDirectCallee()->getName().str();
2214 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
2215 return RValue::get(
Builder.CreateBitOrPointerCast(NewCall,
2221 case Builtin::BIenqueue_kernel: {
2223 unsigned NumArgs = E->getNumArgs();
2225 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
2226 llvm::Type *RangeTy = ConvertType(getContext().OCLNDRangeTy);
2228 llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
2229 llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
2230 llvm::Value *Range = EmitScalarExpr(E->getArg(2));
2235 Name =
"__enqueue_kernel_basic";
2236 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, Int8PtrTy};
2237 llvm::FunctionType *FTy = llvm::FunctionType::get(
2241 Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy);
2243 return RValue::get(
Builder.CreateCall(
2244 CGM.CreateRuntimeFunction(FTy, Name), {Queue, Flags, Range, Block}));
2246 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
2251 Name =
"__enqueue_kernel_vaargs";
2253 Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy);
2256 std::vector<llvm::Value *> Args = {Queue, Flags, Range, Block,
2257 ConstantInt::get(IntTy, NumArgs - 4)};
2258 std::vector<llvm::Type *> ArgTys = {QueueTy, IntTy, RangeTy, Int8PtrTy,
2264 unsigned TypeSizeInBytes =
2266 .getTypeSizeInChars(E->getArg(
I)->
getType())
2268 Args.push_back(TypeSizeInBytes < 4
2269 ?
Builder.CreateZExt(ArgSize, Int32Ty)
2273 llvm::FunctionType *FTy = llvm::FunctionType::get(
2276 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2281 llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
2286 llvm::Type *EventPtrAS4Ty =
2287 EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS4));
2290 llvm::Type *EventPtrAS5Ty =
2291 EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS5));
2293 llvm::Value *NumEvents = EmitScalarExpr(E->getArg(3));
2296 ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
2297 : EmitScalarExpr(E->
getArg(4));
2298 llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
2300 Builder.CreateBitCast(EmitScalarExpr(E->getArg(6)), Int8PtrTy);
2302 std::vector<llvm::Type *> ArgTys = {
2303 QueueTy, Int32Ty, RangeTy, Int32Ty,
2304 EventPtrAS4Ty, EventPtrAS5Ty, Int8PtrTy};
2305 std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
2306 EventList, ClkEvent, Block};
2310 Name =
"__enqueue_kernel_basic_events";
2311 llvm::FunctionType *FTy = llvm::FunctionType::get(
2314 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2319 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
2320 ArgTys.push_back(Int32Ty);
2321 Name =
"__enqueue_kernel_events_vaargs";
2326 unsigned TypeSizeInBytes =
2328 .getTypeSizeInChars(E->getArg(
I)->
getType())
2330 Args.push_back(TypeSizeInBytes < 4
2331 ?
Builder.CreateZExt(ArgSize, Int32Ty)
2334 llvm::FunctionType *FTy = llvm::FunctionType::get(
2337 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2343 case Builtin::BIget_kernel_work_group_size: {
2344 Value *Arg = EmitScalarExpr(E->getArg(0));
2345 Arg =
Builder.CreateBitCast(Arg, Int8PtrTy);
2347 Builder.CreateCall(CGM.CreateRuntimeFunction(
2348 llvm::FunctionType::get(IntTy, Int8PtrTy,
false),
2349 "__get_kernel_work_group_size_impl"),
2352 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
2353 Value *Arg = EmitScalarExpr(E->getArg(0));
2354 Arg =
Builder.CreateBitCast(Arg, Int8PtrTy);
2355 return RValue::get(
Builder.CreateCall(
2356 CGM.CreateRuntimeFunction(
2357 llvm::FunctionType::get(IntTy, Int8PtrTy,
false),
2358 "__get_kernel_preferred_work_group_multiple_impl"),
2361 case Builtin::BIprintf:
2362 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
2363 return EmitCUDADevicePrintfCallExpr(E, ReturnValue);
2365 case Builtin::BI__builtin_canonicalize:
2366 case Builtin::BI__builtin_canonicalizef:
2367 case Builtin::BI__builtin_canonicalizel:
2370 case Builtin::BI__builtin_thread_pointer: {
2371 if (!getContext().getTargetInfo().isTLSSupported())
2372 CGM.ErrorUnsupported(E,
"__builtin_thread_pointer");
2381 if (getContext().
BuiltinInfo.isLibFunction(BuiltinID))
2383 CGM.getBuiltinLibFunction(FD, BuiltinID));
2387 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
2395 checkTargetFeatures(E, FD);
2398 const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
2400 if (
const char *Prefix =
2401 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) {
2402 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name);
2406 if (IntrinsicID == Intrinsic::not_intrinsic)
2407 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name);
2410 if (IntrinsicID != Intrinsic::not_intrinsic) {
2415 unsigned ICEArguments = 0;
2417 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2418 assert(Error == ASTContext::GE_None &&
"Should not codegen an error");
2420 Function *F = CGM.getIntrinsic(IntrinsicID);
2421 llvm::FunctionType *FTy = F->getFunctionType();
2423 for (
unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
2426 if ((ICEArguments & (1 << i)) == 0) {
2427 ArgValue = EmitScalarExpr(E->getArg(i));
2431 llvm::APSInt Result;
2433 assert(IsConst &&
"Constant arg isn't actually constant?");
2435 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
2440 llvm::Type *PTy = FTy->getParamType(i);
2441 if (PTy != ArgValue->getType()) {
2442 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
2443 "Must be able to losslessly bit cast to param");
2444 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
2447 Args.push_back(ArgValue);
2453 llvm::Type *RetTy = VoidTy;
2455 RetTy = ConvertType(BuiltinRetType);
2457 if (RetTy != V->getType()) {
2458 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
2459 "Must be able to losslessly bit cast result type");
2460 V =
Builder.CreateBitCast(V, RetTy);
2463 return RValue::get(V);
2467 if (
Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
2468 return RValue::get(V);
2470 ErrorUnsupported(E,
"builtin function");
2473 return GetUndefRValue(E->
getType());
2477 unsigned BuiltinID,
const CallExpr *E,
2478 llvm::Triple::ArchType Arch) {
2480 case llvm::Triple::arm:
2481 case llvm::Triple::armeb:
2482 case llvm::Triple::thumb:
2483 case llvm::Triple::thumbeb:
2485 case llvm::Triple::aarch64:
2486 case llvm::Triple::aarch64_be:
2488 case llvm::Triple::x86:
2489 case llvm::Triple::x86_64:
2491 case llvm::Triple::ppc:
2492 case llvm::Triple::ppc64:
2493 case llvm::Triple::ppc64le:
2495 case llvm::Triple::r600:
2496 case llvm::Triple::amdgcn:
2498 case llvm::Triple::systemz:
2500 case llvm::Triple::nvptx:
2501 case llvm::Triple::nvptx64:
2503 case llvm::Triple::wasm32:
2504 case llvm::Triple::wasm64:
2511 Value *CodeGenFunction::EmitTargetBuiltinExpr(
unsigned BuiltinID,
2514 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
2527 int IsQuad = TypeFlags.
isQuad();
2531 return llvm::VectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
2535 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
2537 return llvm::VectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
2540 return llvm::VectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
2545 return llvm::VectorType::get(CGF->
Int8Ty, 16);
2547 return llvm::VectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
2549 return llvm::VectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
2551 llvm_unreachable(
"Unknown vector element type!");
2556 int IsQuad = IntTypeFlags.
isQuad();
2559 return llvm::VectorType::get(CGF->
FloatTy, (2 << IsQuad));
2561 return llvm::VectorType::get(CGF->
DoubleTy, (1 << IsQuad));
2563 llvm_unreachable(
"Type can't be converted to floating-point!");
2568 unsigned nElts = V->getType()->getVectorNumElements();
2569 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
2570 return Builder.CreateShuffleVector(V, V, SV,
"lane");
2575 unsigned shift,
bool rightshift) {
2577 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
2578 ai != ae; ++ai, ++j)
2579 if (shift > 0 && shift == j)
2584 return Builder.CreateCall(F, Ops, name);
2589 int SV = cast<ConstantInt>(V)->getSExtValue();
2590 return ConstantInt::get(Ty, neg ? -SV : SV);
2597 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
2599 int ShiftAmt = cast<ConstantInt>(
Shift)->getSExtValue();
2600 int EltSize = VTy->getScalarSizeInBits();
2606 if (ShiftAmt == EltSize) {
2609 return llvm::ConstantAggregateZero::get(VTy);
2614 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
2620 return Builder.CreateLShr(Vec, Shift, name);
2622 return Builder.CreateAShr(Vec, Shift, name);
2648 struct NeonIntrinsicInfo {
2649 const char *NameHint;
2651 unsigned LLVMIntrinsic;
2652 unsigned AltLLVMIntrinsic;
2653 unsigned TypeModifier;
2655 bool operator<(
unsigned RHSBuiltinID)
const {
2656 return BuiltinID < RHSBuiltinID;
2658 bool operator<(
const NeonIntrinsicInfo &TE)
const {
2659 return BuiltinID < TE.BuiltinID;
2664 #define NEONMAP0(NameBase) \
2665 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
2667 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
2668 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
2669 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
2671 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
2672 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
2673 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
2679 NEONMAP1(vabs_v, arm_neon_vabs, 0),
2680 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
2682 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
2683 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
2684 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
2685 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
2688 NEONMAP1(vcage_v, arm_neon_vacge, 0),
2689 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
2690 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
2691 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
2692 NEONMAP1(vcale_v, arm_neon_vacge, 0),
2693 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
2694 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
2695 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
2702 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
2703 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
2705 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
2706 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
2707 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
2708 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
2709 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
2714 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
2715 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
2716 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
2717 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
2718 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
2719 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
2720 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
2721 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
2722 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
2723 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
2724 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
2725 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
2726 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
2727 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
2728 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
2729 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
2730 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
2731 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
2732 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
2733 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
2734 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
2735 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
2736 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
2737 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
2738 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
2739 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
2740 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
2741 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
2742 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
2743 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
2744 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
2745 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
2747 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
2748 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
2749 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
2750 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
2751 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
2765 NEONMAP1(vld1_v, arm_neon_vld1, 0),
2767 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
2768 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
2769 NEONMAP1(vld2_v, arm_neon_vld2, 0),
2770 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
2771 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
2772 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
2773 NEONMAP1(vld3_v, arm_neon_vld3, 0),
2774 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
2775 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
2776 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
2777 NEONMAP1(vld4_v, arm_neon_vld4, 0),
2778 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
2779 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
2805 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
2806 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
2822 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
2823 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
2827 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
2828 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
2849 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
2850 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
2854 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
2855 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
2856 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
2857 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
2858 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
2859 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
2868 NEONMAP1(vst1_v, arm_neon_vst1, 0),
2869 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
2870 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
2871 NEONMAP1(vst2_v, arm_neon_vst2, 0),
2872 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
2873 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
2874 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
2875 NEONMAP1(vst3_v, arm_neon_vst3, 0),
2876 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
2877 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
2878 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
2879 NEONMAP1(vst4_v, arm_neon_vst4, 0),
2880 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
2881 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
2894 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
2895 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
2897 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
2898 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
2899 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
2900 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
2901 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
2902 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
2903 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
2904 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
2905 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
2906 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
2907 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
2908 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
2915 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
2916 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
2918 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2919 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2920 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
2921 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
2922 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
2923 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
2925 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2926 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2927 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
2928 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
2929 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
2930 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
2952 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
2953 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
2969 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
2970 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
2974 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
2975 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
2984 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
2985 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
2989 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
2990 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
2991 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
2992 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
2993 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
2994 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
3057 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
3078 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
3106 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
3183 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
3184 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
3185 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
3186 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
3213 static const NeonIntrinsicInfo *
3215 unsigned BuiltinID,
bool &MapProvenSorted) {
3218 if (!MapProvenSorted) {
3220 MapProvenSorted =
true;
3224 const NeonIntrinsicInfo *Builtin =
3225 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
3227 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
3248 Ty = llvm::VectorType::get(
3249 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
3256 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
3257 ArgType = llvm::VectorType::get(ArgType, Elts);
3261 Tys.push_back(ArgType);
3264 Tys.push_back(ArgType);
3273 const NeonIntrinsicInfo &SISDInfo,
3276 unsigned BuiltinID = SISDInfo.BuiltinID;
3277 unsigned int Int = SISDInfo.LLVMIntrinsic;
3278 unsigned Modifier = SISDInfo.TypeModifier;
3279 const char *s = SISDInfo.NameHint;
3281 switch (BuiltinID) {
3282 case NEON::BI__builtin_neon_vcled_s64:
3283 case NEON::BI__builtin_neon_vcled_u64:
3284 case NEON::BI__builtin_neon_vcles_f32:
3285 case NEON::BI__builtin_neon_vcled_f64:
3286 case NEON::BI__builtin_neon_vcltd_s64:
3287 case NEON::BI__builtin_neon_vcltd_u64:
3288 case NEON::BI__builtin_neon_vclts_f32:
3289 case NEON::BI__builtin_neon_vcltd_f64:
3290 case NEON::BI__builtin_neon_vcales_f32:
3291 case NEON::BI__builtin_neon_vcaled_f64:
3292 case NEON::BI__builtin_neon_vcalts_f32:
3293 case NEON::BI__builtin_neon_vcaltd_f64:
3297 std::swap(Ops[0], Ops[1]);
3301 assert(Int &&
"Generic code assumes a valid intrinsic");
3309 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
3310 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3311 ai != ae; ++ai, ++j) {
3313 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
3314 ArgTy->getPrimitiveSizeInBits())
3317 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
3321 CGF.
Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
3323 CGF.
Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
3328 if (ResultType->getPrimitiveSizeInBits() <
3329 Result->getType()->getPrimitiveSizeInBits())
3330 return CGF.
Builder.CreateExtractElement(Result, C0);
3336 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
3340 llvm::APSInt NeonTypeConst;
3347 bool Usgn = Type.isUnsigned();
3348 bool Quad = Type.isQuad();
3355 auto getAlignmentValue32 = [&](Address addr) ->
Value* {
3356 return Builder.getInt32(addr.getAlignment().getQuantity());
3359 unsigned Int = LLVMIntrinsic;
3361 Int = AltLLVMIntrinsic;
3363 switch (BuiltinID) {
3365 case NEON::BI__builtin_neon_vabs_v:
3366 case NEON::BI__builtin_neon_vabsq_v:
3367 if (VTy->getElementType()->isFloatingPointTy())
3370 case NEON::BI__builtin_neon_vaddhn_v: {
3371 llvm::VectorType *SrcTy =
3372 llvm::VectorType::getExtendedElementVectorType(VTy);
3377 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
3380 Constant *ShiftAmt =
3381 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
3382 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
3385 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
3387 case NEON::BI__builtin_neon_vcale_v:
3388 case NEON::BI__builtin_neon_vcaleq_v:
3389 case NEON::BI__builtin_neon_vcalt_v:
3390 case NEON::BI__builtin_neon_vcaltq_v:
3391 std::swap(Ops[0], Ops[1]);
3392 case NEON::BI__builtin_neon_vcage_v:
3393 case NEON::BI__builtin_neon_vcageq_v:
3394 case NEON::BI__builtin_neon_vcagt_v:
3395 case NEON::BI__builtin_neon_vcagtq_v: {
3398 VTy->getNumElements());
3403 case NEON::BI__builtin_neon_vclz_v:
3404 case NEON::BI__builtin_neon_vclzq_v:
3409 case NEON::BI__builtin_neon_vcvt_f32_v:
3410 case NEON::BI__builtin_neon_vcvtq_f32_v:
3413 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
3414 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
3415 case NEON::BI__builtin_neon_vcvt_n_f32_v:
3416 case NEON::BI__builtin_neon_vcvt_n_f64_v:
3417 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
3418 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
3420 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
3424 case NEON::BI__builtin_neon_vcvt_n_s32_v:
3425 case NEON::BI__builtin_neon_vcvt_n_u32_v:
3426 case NEON::BI__builtin_neon_vcvt_n_s64_v:
3427 case NEON::BI__builtin_neon_vcvt_n_u64_v:
3428 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
3429 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
3430 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
3431 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
3436 case NEON::BI__builtin_neon_vcvt_s32_v:
3437 case NEON::BI__builtin_neon_vcvt_u32_v:
3438 case NEON::BI__builtin_neon_vcvt_s64_v:
3439 case NEON::BI__builtin_neon_vcvt_u64_v:
3440 case NEON::BI__builtin_neon_vcvtq_s32_v:
3441 case NEON::BI__builtin_neon_vcvtq_u32_v:
3442 case NEON::BI__builtin_neon_vcvtq_s64_v:
3443 case NEON::BI__builtin_neon_vcvtq_u64_v: {
3445 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
3446 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
3448 case NEON::BI__builtin_neon_vcvta_s32_v:
3449 case NEON::BI__builtin_neon_vcvta_s64_v:
3450 case NEON::BI__builtin_neon_vcvta_u32_v:
3451 case NEON::BI__builtin_neon_vcvta_u64_v:
3452 case NEON::BI__builtin_neon_vcvtaq_s32_v:
3453 case NEON::BI__builtin_neon_vcvtaq_s64_v:
3454 case NEON::BI__builtin_neon_vcvtaq_u32_v:
3455 case NEON::BI__builtin_neon_vcvtaq_u64_v:
3456 case NEON::BI__builtin_neon_vcvtn_s32_v:
3457 case NEON::BI__builtin_neon_vcvtn_s64_v:
3458 case NEON::BI__builtin_neon_vcvtn_u32_v:
3459 case NEON::BI__builtin_neon_vcvtn_u64_v:
3460 case NEON::BI__builtin_neon_vcvtnq_s32_v:
3461 case NEON::BI__builtin_neon_vcvtnq_s64_v:
3462 case NEON::BI__builtin_neon_vcvtnq_u32_v:
3463 case NEON::BI__builtin_neon_vcvtnq_u64_v:
3464 case NEON::BI__builtin_neon_vcvtp_s32_v:
3465 case NEON::BI__builtin_neon_vcvtp_s64_v:
3466 case NEON::BI__builtin_neon_vcvtp_u32_v:
3467 case NEON::BI__builtin_neon_vcvtp_u64_v:
3468 case NEON::BI__builtin_neon_vcvtpq_s32_v:
3469 case NEON::BI__builtin_neon_vcvtpq_s64_v:
3470 case NEON::BI__builtin_neon_vcvtpq_u32_v:
3471 case NEON::BI__builtin_neon_vcvtpq_u64_v:
3472 case NEON::BI__builtin_neon_vcvtm_s32_v:
3473 case NEON::BI__builtin_neon_vcvtm_s64_v:
3474 case NEON::BI__builtin_neon_vcvtm_u32_v:
3475 case NEON::BI__builtin_neon_vcvtm_u64_v:
3476 case NEON::BI__builtin_neon_vcvtmq_s32_v:
3477 case NEON::BI__builtin_neon_vcvtmq_s64_v:
3478 case NEON::BI__builtin_neon_vcvtmq_u32_v:
3479 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
3483 case NEON::BI__builtin_neon_vext_v:
3484 case NEON::BI__builtin_neon_vextq_v: {
3485 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
3487 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
3488 Indices.push_back(i+CV);
3492 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
3494 case NEON::BI__builtin_neon_vfma_v:
3495 case NEON::BI__builtin_neon_vfmaq_v: {
3502 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
3504 case NEON::BI__builtin_neon_vld1_v:
3505 case NEON::BI__builtin_neon_vld1q_v: {
3507 Ops.push_back(getAlignmentValue32(PtrOp0));
3510 case NEON::BI__builtin_neon_vld2_v:
3511 case NEON::BI__builtin_neon_vld2q_v:
3512 case NEON::BI__builtin_neon_vld3_v:
3513 case NEON::BI__builtin_neon_vld3q_v:
3514 case NEON::BI__builtin_neon_vld4_v:
3515 case NEON::BI__builtin_neon_vld4q_v: {
3518 Value *Align = getAlignmentValue32(PtrOp1);
3519 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
3520 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3524 case NEON::BI__builtin_neon_vld1_dup_v:
3525 case NEON::BI__builtin_neon_vld1q_dup_v: {
3526 Value *V = UndefValue::get(Ty);
3527 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
3530 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
3531 Ops[0] =
Builder.CreateInsertElement(V, Ld, CI);
3534 case NEON::BI__builtin_neon_vld2_lane_v:
3535 case NEON::BI__builtin_neon_vld2q_lane_v:
3536 case NEON::BI__builtin_neon_vld3_lane_v:
3537 case NEON::BI__builtin_neon_vld3q_lane_v:
3538 case NEON::BI__builtin_neon_vld4_lane_v:
3539 case NEON::BI__builtin_neon_vld4q_lane_v: {
3542 for (
unsigned I = 2;
I < Ops.size() - 1; ++
I)
3544 Ops.push_back(getAlignmentValue32(PtrOp1));
3545 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
3546 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3550 case NEON::BI__builtin_neon_vmovl_v: {
3551 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
3554 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
3555 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
3557 case NEON::BI__builtin_neon_vmovn_v: {
3558 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
3560 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
3562 case NEON::BI__builtin_neon_vmull_v:
3568 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
3569 Int = Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
3571 case NEON::BI__builtin_neon_vpadal_v:
3572 case NEON::BI__builtin_neon_vpadalq_v: {
3574 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
3578 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
3582 case NEON::BI__builtin_neon_vpaddl_v:
3583 case NEON::BI__builtin_neon_vpaddlq_v: {
3585 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
3588 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
3592 case NEON::BI__builtin_neon_vqdmlal_v:
3593 case NEON::BI__builtin_neon_vqdmlsl_v: {
3600 case NEON::BI__builtin_neon_vqshl_n_v:
3601 case NEON::BI__builtin_neon_vqshlq_n_v:
3604 case NEON::BI__builtin_neon_vqshlu_n_v:
3605 case NEON::BI__builtin_neon_vqshluq_n_v:
3608 case NEON::BI__builtin_neon_vrecpe_v:
3609 case NEON::BI__builtin_neon_vrecpeq_v:
3610 case NEON::BI__builtin_neon_vrsqrte_v:
3611 case NEON::BI__builtin_neon_vrsqrteq_v:
3612 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
3615 case NEON::BI__builtin_neon_vrshr_n_v:
3616 case NEON::BI__builtin_neon_vrshrq_n_v:
3619 case NEON::BI__builtin_neon_vshl_n_v:
3620 case NEON::BI__builtin_neon_vshlq_n_v:
3624 case NEON::BI__builtin_neon_vshll_n_v: {
3625 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
3628 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
3630 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
3632 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
3634 case NEON::BI__builtin_neon_vshrn_n_v: {
3635 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
3639 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
3641 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
3642 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
3644 case NEON::BI__builtin_neon_vshr_n_v:
3645 case NEON::BI__builtin_neon_vshrq_n_v:
3647 case NEON::BI__builtin_neon_vst1_v:
3648 case NEON::BI__builtin_neon_vst1q_v:
3649 case NEON::BI__builtin_neon_vst2_v:
3650 case NEON::BI__builtin_neon_vst2q_v:
3651 case NEON::BI__builtin_neon_vst3_v:
3652 case NEON::BI__builtin_neon_vst3q_v:
3653 case NEON::BI__builtin_neon_vst4_v:
3654 case NEON::BI__builtin_neon_vst4q_v:
3655 case NEON::BI__builtin_neon_vst2_lane_v:
3656 case NEON::BI__builtin_neon_vst2q_lane_v:
3657 case NEON::BI__builtin_neon_vst3_lane_v:
3658 case NEON::BI__builtin_neon_vst3q_lane_v:
3659 case NEON::BI__builtin_neon_vst4_lane_v:
3660 case NEON::BI__builtin_neon_vst4q_lane_v: {
3662 Ops.push_back(getAlignmentValue32(PtrOp0));
3665 case NEON::BI__builtin_neon_vsubhn_v: {
3666 llvm::VectorType *SrcTy =
3667 llvm::VectorType::getExtendedElementVectorType(VTy);
3672 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
3675 Constant *ShiftAmt =
3676 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
3677 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
3680 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
3682 case NEON::BI__builtin_neon_vtrn_v:
3683 case NEON::BI__builtin_neon_vtrnq_v: {
3687 Value *SV =
nullptr;
3689 for (
unsigned vi = 0; vi != 2; ++vi) {
3691 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
3692 Indices.push_back(i+vi);
3693 Indices.push_back(i+e+vi);
3695 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
3696 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
3701 case NEON::BI__builtin_neon_vtst_v:
3702 case NEON::BI__builtin_neon_vtstq_v: {
3705 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
3706 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
3707 ConstantAggregateZero::get(Ty));
3708 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
3710 case NEON::BI__builtin_neon_vuzp_v:
3711 case NEON::BI__builtin_neon_vuzpq_v: {
3715 Value *SV =
nullptr;
3717 for (
unsigned vi = 0; vi != 2; ++vi) {
3719 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
3720 Indices.push_back(2*i+vi);
3722 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
3723 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
3728 case NEON::BI__builtin_neon_vzip_v:
3729 case NEON::BI__builtin_neon_vzipq_v: {
3733 Value *SV =
nullptr;
3735 for (
unsigned vi = 0; vi != 2; ++vi) {
3737 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
3738 Indices.push_back((i + vi*e) >> 1);
3739 Indices.push_back(((i + vi*e) >> 1)+e);
3741 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
3742 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
3749 assert(Int &&
"Expected valid intrinsic number");
3763 const CmpInst::Predicate Ip,
const Twine &Name) {
3770 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
3771 OTy = BI->getOperand(0)->getType();
3774 if (OTy->getScalarType()->isFloatingPointTy()) {
3775 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
3777 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
3779 return Builder.CreateSExt(Op, Ty, Name);
3788 TblOps.push_back(ExtOp);
3792 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
3793 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
3794 Indices.push_back(2*i);
3795 Indices.push_back(2*i+1);
3798 int PairPos = 0, End = Ops.size() - 1;
3799 while (PairPos < End) {
3800 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
3801 Ops[PairPos+1], Indices,
3808 if (PairPos == End) {
3809 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
3810 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
3811 ZeroTbl, Indices, Name));
3815 TblOps.push_back(IndexOp);
3821 Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
3823 switch (BuiltinID) {
3826 case ARM::BI__builtin_arm_nop:
3829 case ARM::BI__builtin_arm_yield:
3830 case ARM::BI__yield:
3833 case ARM::BI__builtin_arm_wfe:
3837 case ARM::BI__builtin_arm_wfi:
3841 case ARM::BI__builtin_arm_sev:
3845 case ARM::BI__builtin_arm_sevl:
3852 llvm::ConstantInt::get(
Int32Ty, Value));
3863 StringRef SysReg =
"") {
3865 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
3866 &&
"Unsupported size for register.");
3872 if (SysReg.empty()) {
3874 SysReg = cast<StringLiteral>(SysRegStrExpr)->getString();
3877 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
3878 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
3879 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
3883 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
3884 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
3885 &&
"Can't fit 64-bit value in 32-bit register");
3889 llvm::Value *Call = Builder.CreateCall(F, Metadata);
3893 return Builder.CreateTrunc(Call, ValueType);
3895 if (ValueType->isPointerTy())
3897 return Builder.CreateIntToPtr(Call, ValueType);
3906 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
3907 return Builder.CreateCall(F, { Metadata, ArgValue });
3910 if (ValueType->isPointerTy()) {
3912 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
3913 return Builder.CreateCall(F, { Metadata, ArgValue });
3916 return Builder.CreateCall(F, { Metadata, ArgValue });
3922 switch (BuiltinID) {
3924 case NEON::BI__builtin_neon_vget_lane_i8:
3925 case NEON::BI__builtin_neon_vget_lane_i16:
3926 case NEON::BI__builtin_neon_vget_lane_i32:
3927 case NEON::BI__builtin_neon_vget_lane_i64:
3928 case NEON::BI__builtin_neon_vget_lane_f32:
3929 case NEON::BI__builtin_neon_vgetq_lane_i8:
3930 case NEON::BI__builtin_neon_vgetq_lane_i16:
3931 case NEON::BI__builtin_neon_vgetq_lane_i32:
3932 case NEON::BI__builtin_neon_vgetq_lane_i64:
3933 case NEON::BI__builtin_neon_vgetq_lane_f32:
3934 case NEON::BI__builtin_neon_vset_lane_i8:
3935 case NEON::BI__builtin_neon_vset_lane_i16:
3936 case NEON::BI__builtin_neon_vset_lane_i32:
3937 case NEON::BI__builtin_neon_vset_lane_i64:
3938 case NEON::BI__builtin_neon_vset_lane_f32:
3939 case NEON::BI__builtin_neon_vsetq_lane_i8:
3940 case NEON::BI__builtin_neon_vsetq_lane_i16:
3941 case NEON::BI__builtin_neon_vsetq_lane_i32:
3942 case NEON::BI__builtin_neon_vsetq_lane_i64:
3943 case NEON::BI__builtin_neon_vsetq_lane_f32:
3944 case NEON::BI__builtin_neon_vsha1h_u32:
3945 case NEON::BI__builtin_neon_vsha1cq_u32:
3946 case NEON::BI__builtin_neon_vsha1pq_u32:
3947 case NEON::BI__builtin_neon_vsha1mq_u32:
3948 case ARM::BI_MoveToCoprocessor:
3949 case ARM::BI_MoveToCoprocessor2:
3957 if (
auto Hint = GetValueForARMHint(BuiltinID))
3960 if (BuiltinID == ARM::BI__emit) {
3962 llvm::FunctionType *FTy =
3963 llvm::FunctionType::get(
VoidTy,
false);
3967 llvm_unreachable(
"Sema will ensure that the parameter is constant");
3969 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
3971 llvm::InlineAsm *Emit =
3972 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
3974 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
3977 return Builder.CreateCall(Emit);
3980 if (BuiltinID == ARM::BI__builtin_arm_dbg) {
3985 if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
3991 Value *Locality = llvm::ConstantInt::get(
Int32Ty, 3);
3994 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
3997 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
4003 if (BuiltinID == ARM::BI__clear_cache) {
4004 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
4007 for (
unsigned i = 0; i < 2; i++)
4010 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
4011 StringRef Name = FD->getName();
4015 if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
4016 BuiltinID == ARM::BI__builtin_arm_mcrr2) {
4019 switch (BuiltinID) {
4020 default: llvm_unreachable(
"unexpected builtin");
4021 case ARM::BI__builtin_arm_mcrr:
4024 case ARM::BI__builtin_arm_mcrr2:
4041 Value *C1 = llvm::ConstantInt::get(
Int64Ty, 32);
4043 Value *Rt2 =
Builder.CreateLShr(RtAndRt2, C1);
4046 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
4049 if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
4050 BuiltinID == ARM::BI__builtin_arm_mrrc2) {
4053 switch (BuiltinID) {
4054 default: llvm_unreachable(
"unexpected builtin");
4055 case ARM::BI__builtin_arm_mrrc:
4058 case ARM::BI__builtin_arm_mrrc2:
4066 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
4071 Value *Rt =
Builder.CreateExtractValue(RtAndRt2, 1);
4072 Value *Rt1 =
Builder.CreateExtractValue(RtAndRt2, 0);
4076 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
4077 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
4078 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
4083 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
4084 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
4085 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
4087 BuiltinID == ARM::BI__ldrexd) {
4090 switch (BuiltinID) {
4091 default: llvm_unreachable(
"unexpected builtin");
4092 case ARM::BI__builtin_arm_ldaex:
4095 case ARM::BI__builtin_arm_ldrexd:
4096 case ARM::BI__builtin_arm_ldrex:
4097 case ARM::BI__ldrexd:
4106 Value *Val0 =
Builder.CreateExtractValue(Val, 1);
4107 Value *Val1 =
Builder.CreateExtractValue(Val, 0);
4111 Value *ShiftCst = llvm::ConstantInt::get(
Int64Ty, 32);
4112 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
4113 Val =
Builder.CreateOr(Val, Val1);
4117 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
4118 BuiltinID == ARM::BI__builtin_arm_ldaex) {
4127 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
4128 ? Intrinsic::arm_ldaex
4129 : Intrinsic::arm_ldrex,
4130 LoadAddr->getType());
4131 Value *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
4133 if (RealResTy->isPointerTy())
4134 return Builder.CreateIntToPtr(Val, RealResTy);
4136 Val =
Builder.CreateTruncOrBitCast(Val, IntResTy);
4141 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
4142 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
4143 BuiltinID == ARM::BI__builtin_arm_strex) &&
4145 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4146 ? Intrinsic::arm_stlexd
4147 : Intrinsic::arm_strexd);
4157 Value *Arg0 =
Builder.CreateExtractValue(Val, 0);
4158 Value *Arg1 =
Builder.CreateExtractValue(Val, 1);
4160 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
4163 if (BuiltinID == ARM::BI__builtin_arm_strex ||
4164 BuiltinID == ARM::BI__builtin_arm_stlex) {
4173 if (StoreVal->getType()->isPointerTy())
4180 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4181 ? Intrinsic::arm_stlex
4182 : Intrinsic::arm_strex,
4183 StoreAddr->getType());
4184 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
4187 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
4194 switch (BuiltinID) {
4195 case ARM::BI__builtin_arm_crc32b:
4196 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
4197 case ARM::BI__builtin_arm_crc32cb:
4198 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
4199 case ARM::BI__builtin_arm_crc32h:
4200 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
4201 case ARM::BI__builtin_arm_crc32ch:
4202 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
4203 case ARM::BI__builtin_arm_crc32w:
4204 case ARM::BI__builtin_arm_crc32d:
4205 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
4206 case ARM::BI__builtin_arm_crc32cw:
4207 case ARM::BI__builtin_arm_crc32cd:
4208 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
4211 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4217 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
4218 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
4219 Value *C1 = llvm::ConstantInt::get(
Int64Ty, 32);
4221 Value *Arg1b =
Builder.CreateLShr(Arg1, C1);
4225 Value *Res =
Builder.CreateCall(F, {Arg0, Arg1a});
4226 return Builder.CreateCall(F, {Res, Arg1b});
4231 return Builder.CreateCall(F, {Arg0, Arg1});
4235 if (BuiltinID == ARM::BI__builtin_arm_rsr ||
4236 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4237 BuiltinID == ARM::BI__builtin_arm_rsrp ||
4238 BuiltinID == ARM::BI__builtin_arm_wsr ||
4239 BuiltinID == ARM::BI__builtin_arm_wsr64 ||
4240 BuiltinID == ARM::BI__builtin_arm_wsrp) {
4242 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
4243 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4244 BuiltinID == ARM::BI__builtin_arm_rsrp;
4246 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
4247 BuiltinID == ARM::BI__builtin_arm_wsrp;
4249 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4250 BuiltinID == ARM::BI__builtin_arm_wsr64;
4254 if (IsPointerBuiltin) {
4257 }
else if (Is64Bit) {
4258 ValueType = RegisterType =
Int64Ty;
4260 ValueType = RegisterType =
Int32Ty;
4268 unsigned ICEArguments = 0;
4273 auto getAlignmentValue32 = [&](Address addr) -> Value* {
4274 return Builder.getInt32(addr.getAlignment().getQuantity());
4281 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
4282 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
4284 switch (BuiltinID) {
4285 case NEON::BI__builtin_neon_vld1_v:
4286 case NEON::BI__builtin_neon_vld1q_v:
4287 case NEON::BI__builtin_neon_vld1q_lane_v:
4288 case NEON::BI__builtin_neon_vld1_lane_v:
4289 case NEON::BI__builtin_neon_vld1_dup_v:
4290 case NEON::BI__builtin_neon_vld1q_dup_v:
4291 case NEON::BI__builtin_neon_vst1_v:
4292 case NEON::BI__builtin_neon_vst1q_v:
4293 case NEON::BI__builtin_neon_vst1q_lane_v:
4294 case NEON::BI__builtin_neon_vst1_lane_v:
4295 case NEON::BI__builtin_neon_vst2_v:
4296 case NEON::BI__builtin_neon_vst2q_v:
4297 case NEON::BI__builtin_neon_vst2_lane_v:
4298 case NEON::BI__builtin_neon_vst2q_lane_v:
4299 case NEON::BI__builtin_neon_vst3_v:
4300 case NEON::BI__builtin_neon_vst3q_v:
4301 case NEON::BI__builtin_neon_vst3_lane_v:
4302 case NEON::BI__builtin_neon_vst3q_lane_v:
4303 case NEON::BI__builtin_neon_vst4_v:
4304 case NEON::BI__builtin_neon_vst4q_v:
4305 case NEON::BI__builtin_neon_vst4_lane_v:
4306 case NEON::BI__builtin_neon_vst4q_lane_v:
4315 switch (BuiltinID) {
4316 case NEON::BI__builtin_neon_vld2_v:
4317 case NEON::BI__builtin_neon_vld2q_v:
4318 case NEON::BI__builtin_neon_vld3_v:
4319 case NEON::BI__builtin_neon_vld3q_v:
4320 case NEON::BI__builtin_neon_vld4_v:
4321 case NEON::BI__builtin_neon_vld4q_v:
4322 case NEON::BI__builtin_neon_vld2_lane_v:
4323 case NEON::BI__builtin_neon_vld2q_lane_v:
4324 case NEON::BI__builtin_neon_vld3_lane_v:
4325 case NEON::BI__builtin_neon_vld3q_lane_v:
4326 case NEON::BI__builtin_neon_vld4_lane_v:
4327 case NEON::BI__builtin_neon_vld4q_lane_v:
4328 case NEON::BI__builtin_neon_vld2_dup_v:
4329 case NEON::BI__builtin_neon_vld3_dup_v:
4330 case NEON::BI__builtin_neon_vld4_dup_v:
4339 if ((ICEArguments & (1 << i)) == 0) {
4344 llvm::APSInt Result;
4346 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
4351 switch (BuiltinID) {
4354 case NEON::BI__builtin_neon_vget_lane_i8:
4355 case NEON::BI__builtin_neon_vget_lane_i16:
4356 case NEON::BI__builtin_neon_vget_lane_i32:
4357 case NEON::BI__builtin_neon_vget_lane_i64:
4358 case NEON::BI__builtin_neon_vget_lane_f32:
4359 case NEON::BI__builtin_neon_vgetq_lane_i8:
4360 case NEON::BI__builtin_neon_vgetq_lane_i16:
4361 case NEON::BI__builtin_neon_vgetq_lane_i32:
4362 case NEON::BI__builtin_neon_vgetq_lane_i64:
4363 case NEON::BI__builtin_neon_vgetq_lane_f32:
4364 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
4366 case NEON::BI__builtin_neon_vset_lane_i8:
4367 case NEON::BI__builtin_neon_vset_lane_i16:
4368 case NEON::BI__builtin_neon_vset_lane_i32:
4369 case NEON::BI__builtin_neon_vset_lane_i64:
4370 case NEON::BI__builtin_neon_vset_lane_f32:
4371 case NEON::BI__builtin_neon_vsetq_lane_i8:
4372 case NEON::BI__builtin_neon_vsetq_lane_i16:
4373 case NEON::BI__builtin_neon_vsetq_lane_i32:
4374 case NEON::BI__builtin_neon_vsetq_lane_i64:
4375 case NEON::BI__builtin_neon_vsetq_lane_f32:
4376 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
4378 case NEON::BI__builtin_neon_vsha1h_u32:
4381 case NEON::BI__builtin_neon_vsha1cq_u32:
4384 case NEON::BI__builtin_neon_vsha1pq_u32:
4387 case NEON::BI__builtin_neon_vsha1mq_u32:
4393 case ARM::BI_MoveToCoprocessor:
4394 case ARM::BI_MoveToCoprocessor2: {
4395 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
4396 Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
4397 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
4398 Ops[3], Ops[4], Ops[5]});
4403 assert(HasExtraArg);
4404 llvm::APSInt Result;
4409 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
4410 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
4413 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
4419 bool usgn = Result.getZExtValue() == 1;
4420 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
4424 return Builder.CreateCall(F, Ops,
"vcvtr");
4429 bool usgn = Type.isUnsigned();
4430 bool rightShift =
false;
4439 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
4441 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
4444 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
4445 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1);
4448 switch (BuiltinID) {
4449 default:
return nullptr;
4450 case NEON::BI__builtin_neon_vld1q_lane_v:
4453 if (VTy->getElementType()->isIntegerTy(64)) {
4456 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
4457 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
4458 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4460 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
4463 Value *Align = getAlignmentValue32(PtrOp0);
4464 Value *Ld =
Builder.CreateCall(F, {Ops[0], Align});
4466 uint32_t Indices[] = {1 - Lane, Lane};
4468 return Builder.CreateShuffleVector(Ops[1], Ld, SV,
"vld1q_lane");
4471 case NEON::BI__builtin_neon_vld1_lane_v: {
4475 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
4477 case NEON::BI__builtin_neon_vld2_dup_v:
4478 case NEON::BI__builtin_neon_vld3_dup_v:
4479 case NEON::BI__builtin_neon_vld4_dup_v: {
4481 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
4482 switch (BuiltinID) {
4483 case NEON::BI__builtin_neon_vld2_dup_v:
4484 Int = Intrinsic::arm_neon_vld2;
4486 case NEON::BI__builtin_neon_vld3_dup_v:
4487 Int = Intrinsic::arm_neon_vld3;
4489 case NEON::BI__builtin_neon_vld4_dup_v:
4490 Int = Intrinsic::arm_neon_vld4;
4492 default: llvm_unreachable(
"unknown vld_dup intrinsic?");
4497 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align},
"vld_dup");
4498 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4502 switch (BuiltinID) {
4503 case NEON::BI__builtin_neon_vld2_dup_v:
4504 Int = Intrinsic::arm_neon_vld2lane;
4506 case NEON::BI__builtin_neon_vld3_dup_v:
4507 Int = Intrinsic::arm_neon_vld3lane;
4509 case NEON::BI__builtin_neon_vld4_dup_v:
4510 Int = Intrinsic::arm_neon_vld4lane;
4512 default: llvm_unreachable(
"unknown vld_dup intrinsic?");
4516 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
4519 Args.push_back(Ops[1]);
4520 Args.append(STy->getNumElements(), UndefValue::get(Ty));
4522 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
4524 Args.push_back(getAlignmentValue32(PtrOp1));
4526 Ops[1] =
Builder.CreateCall(F, Args,
"vld_dup");
4528 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
4529 Value *Val =
Builder.CreateExtractValue(Ops[1], i);
4533 Ops[1] =
Builder.CreateInsertValue(Ops[1], Elt, i);
4535 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4539 case NEON::BI__builtin_neon_vqrshrn_n_v:
4541 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
4544 case NEON::BI__builtin_neon_vqrshrun_n_v:
4546 Ops,
"vqrshrun_n", 1,
true);
4547 case NEON::BI__builtin_neon_vqshrn_n_v:
4548 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
4551 case NEON::BI__builtin_neon_vqshrun_n_v:
4553 Ops,
"vqshrun_n", 1,
true);
4554 case NEON::BI__builtin_neon_vrecpe_v:
4555 case NEON::BI__builtin_neon_vrecpeq_v:
4558 case NEON::BI__builtin_neon_vrshrn_n_v:
4560 Ops,
"vrshrn_n", 1,
true);
4561 case NEON::BI__builtin_neon_vrsra_n_v:
4562 case NEON::BI__builtin_neon_vrsraq_n_v:
4566 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
4568 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
4569 case NEON::BI__builtin_neon_vsri_n_v:
4570 case NEON::BI__builtin_neon_vsriq_n_v:
4572 case NEON::BI__builtin_neon_vsli_n_v:
4573 case NEON::BI__builtin_neon_vsliq_n_v:
4577 case NEON::BI__builtin_neon_vsra_n_v:
4578 case NEON::BI__builtin_neon_vsraq_n_v:
4581 return Builder.CreateAdd(Ops[0], Ops[1]);
4582 case NEON::BI__builtin_neon_vst1q_lane_v:
4585 if (VTy->getElementType()->isIntegerTy(64)) {
4587 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
4588 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4589 Ops[2] = getAlignmentValue32(PtrOp0);
4595 case NEON::BI__builtin_neon_vst1_lane_v: {
4597 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
4598 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4602 case NEON::BI__builtin_neon_vtbl1_v:
4605 case NEON::BI__builtin_neon_vtbl2_v:
4608 case NEON::BI__builtin_neon_vtbl3_v:
4611 case NEON::BI__builtin_neon_vtbl4_v:
4614 case NEON::BI__builtin_neon_vtbx1_v:
4617 case NEON::BI__builtin_neon_vtbx2_v:
4620 case NEON::BI__builtin_neon_vtbx3_v:
4623 case NEON::BI__builtin_neon_vtbx4_v:
4632 unsigned int Int = 0;
4633 const char *s =
nullptr;
4635 switch (BuiltinID) {
4638 case NEON::BI__builtin_neon_vtbl1_v:
4639 case NEON::BI__builtin_neon_vqtbl1_v:
4640 case NEON::BI__builtin_neon_vqtbl1q_v:
4641 case NEON::BI__builtin_neon_vtbl2_v:
4642 case NEON::BI__builtin_neon_vqtbl2_v:
4643 case NEON::BI__builtin_neon_vqtbl2q_v:
4644 case NEON::BI__builtin_neon_vtbl3_v:
4645 case NEON::BI__builtin_neon_vqtbl3_v:
4646 case NEON::BI__builtin_neon_vqtbl3q_v:
4647 case NEON::BI__builtin_neon_vtbl4_v:
4648 case NEON::BI__builtin_neon_vqtbl4_v:
4649 case NEON::BI__builtin_neon_vqtbl4q_v:
4651 case NEON::BI__builtin_neon_vtbx1_v:
4652 case NEON::BI__builtin_neon_vqtbx1_v:
4653 case NEON::BI__builtin_neon_vqtbx1q_v:
4654 case NEON::BI__builtin_neon_vtbx2_v:
4655 case NEON::BI__builtin_neon_vqtbx2_v:
4656 case NEON::BI__builtin_neon_vqtbx2q_v:
4657 case NEON::BI__builtin_neon_vtbx3_v:
4658 case NEON::BI__builtin_neon_vqtbx3_v:
4659 case NEON::BI__builtin_neon_vqtbx3q_v:
4660 case NEON::BI__builtin_neon_vtbx4_v:
4661 case NEON::BI__builtin_neon_vqtbx4_v:
4662 case NEON::BI__builtin_neon_vqtbx4q_v:
4669 llvm::APSInt Result;
4684 switch (BuiltinID) {
4685 case NEON::BI__builtin_neon_vtbl1_v: {
4687 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
4690 case NEON::BI__builtin_neon_vtbl2_v: {
4692 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
4695 case NEON::BI__builtin_neon_vtbl3_v: {
4697 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
4700 case NEON::BI__builtin_neon_vtbl4_v: {
4702 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
4705 case NEON::BI__builtin_neon_vtbx1_v: {
4708 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
4710 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
4711 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
4712 CmpRes = Builder.CreateSExt(CmpRes, Ty);
4714 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
4715 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
4716 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
4718 case NEON::BI__builtin_neon_vtbx2_v: {
4720 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
4723 case NEON::BI__builtin_neon_vtbx3_v: {
4726 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
4728 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
4729 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
4731 CmpRes = Builder.CreateSExt(CmpRes, Ty);
4733 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
4734 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
4735 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
4737 case NEON::BI__builtin_neon_vtbx4_v: {
4739 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
4742 case NEON::BI__builtin_neon_vqtbl1_v:
4743 case NEON::BI__builtin_neon_vqtbl1q_v:
4744 Int = Intrinsic::aarch64_neon_tbl1; s =
"vtbl1";
break;
4745 case NEON::BI__builtin_neon_vqtbl2_v:
4746 case NEON::BI__builtin_neon_vqtbl2q_v: {
4747 Int = Intrinsic::aarch64_neon_tbl2; s =
"vtbl2";
break;
4748 case NEON::BI__builtin_neon_vqtbl3_v:
4749 case NEON::BI__builtin_neon_vqtbl3q_v:
4750 Int = Intrinsic::aarch64_neon_tbl3; s =
"vtbl3";
break;
4751 case NEON::BI__builtin_neon_vqtbl4_v:
4752 case NEON::BI__builtin_neon_vqtbl4q_v:
4753 Int = Intrinsic::aarch64_neon_tbl4; s =
"vtbl4";
break;
4754 case NEON::BI__builtin_neon_vqtbx1_v:
4755 case NEON::BI__builtin_neon_vqtbx1q_v:
4756 Int = Intrinsic::aarch64_neon_tbx1; s =
"vtbx1";
break;
4757 case NEON::BI__builtin_neon_vqtbx2_v:
4758 case NEON::BI__builtin_neon_vqtbx2q_v:
4759 Int = Intrinsic::aarch64_neon_tbx2; s =
"vtbx2";
break;
4760 case NEON::BI__builtin_neon_vqtbx3_v:
4761 case NEON::BI__builtin_neon_vqtbx3q_v:
4762 Int = Intrinsic::aarch64_neon_tbx3; s =
"vtbx3";
break;
4763 case NEON::BI__builtin_neon_vqtbx4_v:
4764 case NEON::BI__builtin_neon_vqtbx4q_v:
4765 Int = Intrinsic::aarch64_neon_tbx4; s =
"vtbx4";
break;
4779 Value *V = UndefValue::get(VTy);
4780 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
4781 Op =
Builder.CreateInsertElement(V, Op, CI);
4787 unsigned HintID =
static_cast<unsigned>(-1);
4788 switch (BuiltinID) {
4790 case AArch64::BI__builtin_arm_nop:
4793 case AArch64::BI__builtin_arm_yield:
4796 case AArch64::BI__builtin_arm_wfe:
4799 case AArch64::BI__builtin_arm_wfi:
4802 case AArch64::BI__builtin_arm_sev:
4805 case AArch64::BI__builtin_arm_sevl:
4810 if (HintID != static_cast<unsigned>(-1)) {
4812 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
4815 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
4822 Value *Locality =
nullptr;
4823 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
4825 Locality = llvm::ConstantInt::get(
Int32Ty,
4826 -cast<llvm::ConstantInt>(CacheLevel)->
getValue() + 3);
4829 Locality = llvm::ConstantInt::get(
Int32Ty, 0);
4835 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
4838 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
4840 "rbit of unusual size!");
4843 CGM.
getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg,
"rbit");
4845 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
4847 "rbit of unusual size!");
4850 CGM.
getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg,
"rbit");
4853 if (BuiltinID == AArch64::BI__clear_cache) {
4854 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
4857 for (
unsigned i = 0; i < 2; i++)
4860 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
4861 StringRef Name = FD->getName();
4865 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
4866 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
4868 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
4869 ? Intrinsic::aarch64_ldaxp
4870 : Intrinsic::aarch64_ldxp);
4876 Value *Val0 =
Builder.CreateExtractValue(Val, 1);
4877 Value *Val1 =
Builder.CreateExtractValue(Val, 0);
4879 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
4880 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
4882 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
4883 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
4884 Val =
Builder.CreateOr(Val, Val1);
4886 }
else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
4887 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
4896 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
4897 ? Intrinsic::aarch64_ldaxr
4898 : Intrinsic::aarch64_ldxr,
4899 LoadAddr->getType());
4900 Value *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
4902 if (RealResTy->isPointerTy())
4903 return Builder.CreateIntToPtr(Val, RealResTy);
4905 Val =
Builder.CreateTruncOrBitCast(Val, IntResTy);
4909 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
4910 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
4912 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
4913 ? Intrinsic::aarch64_stlxp
4914 : Intrinsic::aarch64_stxp);
4923 Value *Arg0 =
Builder.CreateExtractValue(Val, 0);
4924 Value *Arg1 =
Builder.CreateExtractValue(Val, 1);
4927 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
4930 if (BuiltinID == AArch64::BI__builtin_arm_strex ||
4931 BuiltinID == AArch64::BI__builtin_arm_stlex) {
4940 if (StoreVal->getType()->isPointerTy())
4947 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
4948 ? Intrinsic::aarch64_stlxr
4949 : Intrinsic::aarch64_stxr,
4950 StoreAddr->getType());
4951 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
4954 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
4961 switch (BuiltinID) {
4962 case AArch64::BI__builtin_arm_crc32b:
4963 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
4964 case AArch64::BI__builtin_arm_crc32cb:
4965 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
4966 case AArch64::BI__builtin_arm_crc32h:
4967 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
4968 case AArch64::BI__builtin_arm_crc32ch:
4969 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
4970 case AArch64::BI__builtin_arm_crc32w:
4971 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
4972 case AArch64::BI__builtin_arm_crc32cw:
4973 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
4974 case AArch64::BI__builtin_arm_crc32d:
4975 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
4976 case AArch64::BI__builtin_arm_crc32cd:
4977 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
4980 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4985 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
4986 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
4988 return Builder.CreateCall(F, {Arg0, Arg1});
4991 if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
4992 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
4993 BuiltinID == AArch64::BI__builtin_arm_rsrp ||
4994 BuiltinID == AArch64::BI__builtin_arm_wsr ||
4995 BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
4996 BuiltinID == AArch64::BI__builtin_arm_wsrp) {
4998 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
4999 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5000 BuiltinID == AArch64::BI__builtin_arm_rsrp;
5002 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5003 BuiltinID == AArch64::BI__builtin_arm_wsrp;
5005 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
5006 BuiltinID != AArch64::BI__builtin_arm_wsr;
5010 if (IsPointerBuiltin) {
5012 }
else if (Is64Bit) {
5023 unsigned ICEArguments = 0;
5029 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
5030 if ((ICEArguments & (1 << i)) == 0) {
5035 llvm::APSInt Result;
5037 assert(IsConst &&
"Constant arg isn't actually constant?");
5043 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
5045 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5050 assert(Result &&
"SISD intrinsic should have been handled");
5054 llvm::APSInt Result;
5062 bool quad = Type.
isQuad();
5065 switch (BuiltinID) {
5067 case NEON::BI__builtin_neon_vldrq_p128: {
5072 case NEON::BI__builtin_neon_vstrq_p128: {
5077 case NEON::BI__builtin_neon_vcvts_u32_f32:
5078 case NEON::BI__builtin_neon_vcvtd_u64_f64:
5081 case NEON::BI__builtin_neon_vcvts_s32_f32:
5082 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
5084 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5089 return Builder.CreateFPToUI(Ops[0], InTy);
5090 return Builder.CreateFPToSI(Ops[0], InTy);
5092 case NEON::BI__builtin_neon_vcvts_f32_u32:
5093 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5096 case NEON::BI__builtin_neon_vcvts_f32_s32:
5097 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5099 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5104 return Builder.CreateUIToFP(Ops[0], FTy);
5105 return Builder.CreateSIToFP(Ops[0], FTy);
5107 case NEON::BI__builtin_neon_vpaddd_s64: {
5114 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5115 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5117 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
5119 case NEON::BI__builtin_neon_vpaddd_f64: {
5121 llvm::VectorType::get(
DoubleTy, 2);
5127 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5128 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5130 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5132 case NEON::BI__builtin_neon_vpadds_f32: {
5134 llvm::VectorType::get(
FloatTy, 2);
5140 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5141 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5143 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5145 case NEON::BI__builtin_neon_vceqzd_s64:
5146 case NEON::BI__builtin_neon_vceqzd_f64:
5147 case NEON::BI__builtin_neon_vceqzs_f32:
5151 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
5152 case NEON::BI__builtin_neon_vcgezd_s64:
5153 case NEON::BI__builtin_neon_vcgezd_f64:
5154 case NEON::BI__builtin_neon_vcgezs_f32:
5158 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
5159 case NEON::BI__builtin_neon_vclezd_s64:
5160 case NEON::BI__builtin_neon_vclezd_f64:
5161 case NEON::BI__builtin_neon_vclezs_f32:
5165 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
5166 case NEON::BI__builtin_neon_vcgtzd_s64:
5167 case NEON::BI__builtin_neon_vcgtzd_f64:
5168 case NEON::BI__builtin_neon_vcgtzs_f32:
5172 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
5173 case NEON::BI__builtin_neon_vcltzd_s64:
5174 case NEON::BI__builtin_neon_vcltzd_f64:
5175 case NEON::BI__builtin_neon_vcltzs_f32:
5179 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
5181 case NEON::BI__builtin_neon_vceqzd_u64: {
5185 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
5188 case NEON::BI__builtin_neon_vceqd_f64:
5189 case NEON::BI__builtin_neon_vcled_f64:
5190 case NEON::BI__builtin_neon_vcltd_f64:
5191 case NEON::BI__builtin_neon_vcged_f64:
5192 case NEON::BI__builtin_neon_vcgtd_f64: {
5193 llvm::CmpInst::Predicate
P;
5194 switch (BuiltinID) {
5195 default: llvm_unreachable(
"missing builtin ID in switch!");
5196 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ;
break;
5197 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE;
break;
5198 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT;
break;
5199 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE;
break;
5200 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT;
break;
5205 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5208 case NEON::BI__builtin_neon_vceqs_f32:
5209 case NEON::BI__builtin_neon_vcles_f32:
5210 case NEON::BI__builtin_neon_vclts_f32:
5211 case NEON::BI__builtin_neon_vcges_f32:
5212 case NEON::BI__builtin_neon_vcgts_f32: {
5213 llvm::CmpInst::Predicate
P;
5214 switch (BuiltinID) {
5215 default: llvm_unreachable(
"missing builtin ID in switch!");
5216 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ;
break;
5217 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE;
break;
5218 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT;
break;
5219 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE;
break;
5220 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT;
break;
5225 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5228 case NEON::BI__builtin_neon_vceqd_s64:
5229 case NEON::BI__builtin_neon_vceqd_u64:
5230 case NEON::BI__builtin_neon_vcgtd_s64:
5231 case NEON::BI__builtin_neon_vcgtd_u64:
5232 case NEON::BI__builtin_neon_vcltd_s64:
5233 case NEON::BI__builtin_neon_vcltd_u64:
5234 case NEON::BI__builtin_neon_vcged_u64:
5235 case NEON::BI__builtin_neon_vcged_s64:
5236 case NEON::BI__builtin_neon_vcled_u64:
5237 case NEON::BI__builtin_neon_vcled_s64: {
5238 llvm::CmpInst::Predicate
P;
5239 switch (BuiltinID) {
5240 default: llvm_unreachable(
"missing builtin ID in switch!");
5241 case NEON::BI__builtin_neon_vceqd_s64:
5242 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;
break;
5243 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;
break;
5244 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;
break;
5245 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;
break;
5246 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;
break;
5247 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;
break;
5248 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;
break;
5249 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;
break;
5250 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;
break;
5255 Ops[0] =
Builder.CreateICmp(P, Ops[0], Ops[1]);
5258 case NEON::BI__builtin_neon_vtstd_s64:
5259 case NEON::BI__builtin_neon_vtstd_u64: {
5263 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
5264 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5265 llvm::Constant::getNullValue(
Int64Ty));
5268 case NEON::BI__builtin_neon_vset_lane_i8:
5269 case NEON::BI__builtin_neon_vset_lane_i16:
5270 case NEON::BI__builtin_neon_vset_lane_i32:
5271 case NEON::BI__builtin_neon_vset_lane_i64:
5272 case NEON::BI__builtin_neon_vset_lane_f32:
5273 case NEON::BI__builtin_neon_vsetq_lane_i8:
5274 case NEON::BI__builtin_neon_vsetq_lane_i16:
5275 case NEON::BI__builtin_neon_vsetq_lane_i32:
5276 case NEON::BI__builtin_neon_vsetq_lane_i64:
5277 case NEON::BI__builtin_neon_vsetq_lane_f32:
5279 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5280 case NEON::BI__builtin_neon_vset_lane_f64:
5283 llvm::VectorType::get(
DoubleTy, 1));
5285 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5286 case NEON::BI__builtin_neon_vsetq_lane_f64:
5289 llvm::VectorType::get(
DoubleTy, 2));
5291 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5293 case NEON::BI__builtin_neon_vget_lane_i8:
5294 case NEON::BI__builtin_neon_vdupb_lane_i8:
5298 case NEON::BI__builtin_neon_vgetq_lane_i8:
5299 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5303 case NEON::BI__builtin_neon_vget_lane_i16:
5304 case NEON::BI__builtin_neon_vduph_lane_i16:
5308 case NEON::BI__builtin_neon_vgetq_lane_i16:
5309 case NEON::BI__builtin_neon_vduph_laneq_i16:
5313 case NEON::BI__builtin_neon_vget_lane_i32:
5314 case NEON::BI__builtin_neon_vdups_lane_i32:
5318 case NEON::BI__builtin_neon_vdups_lane_f32:
5320 llvm::VectorType::get(
FloatTy, 2));
5323 case NEON::BI__builtin_neon_vgetq_lane_i32:
5324 case NEON::BI__builtin_neon_vdups_laneq_i32:
5328 case NEON::BI__builtin_neon_vget_lane_i64:
5329 case NEON::BI__builtin_neon_vdupd_lane_i64:
5333 case NEON::BI__builtin_neon_vdupd_lane_f64:
5335 llvm::VectorType::get(
DoubleTy, 1));
5338 case NEON::BI__builtin_neon_vgetq_lane_i64:
5339 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5343 case NEON::BI__builtin_neon_vget_lane_f32:
5345 llvm::VectorType::get(
FloatTy, 2));
5348 case NEON::BI__builtin_neon_vget_lane_f64:
5350 llvm::VectorType::get(
DoubleTy, 1));
5353 case NEON::BI__builtin_neon_vgetq_lane_f32:
5354 case NEON::BI__builtin_neon_vdups_laneq_f32:
5356 llvm::VectorType::get(
FloatTy, 4));
5359 case NEON::BI__builtin_neon_vgetq_lane_f64:
5360 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5362 llvm::VectorType::get(
DoubleTy, 2));
5365 case NEON::BI__builtin_neon_vaddd_s64:
5366 case NEON::BI__builtin_neon_vaddd_u64:
5368 case NEON::BI__builtin_neon_vsubd_s64:
5369 case NEON::BI__builtin_neon_vsubd_u64:
5371 case NEON::BI__builtin_neon_vqdmlalh_s16:
5372 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5378 ProductOps,
"vqdmlXl");
5379 Constant *CI = ConstantInt::get(
SizeTy, 0);
5380 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
5382 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5383 ? Intrinsic::aarch64_neon_sqadd
5384 : Intrinsic::aarch64_neon_sqsub;
5387 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5393 case NEON::BI__builtin_neon_vqshld_n_u64:
5394 case NEON::BI__builtin_neon_vqshld_n_s64: {
5395 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5396 ? Intrinsic::aarch64_neon_uqshl
5397 : Intrinsic::aarch64_neon_sqshl;
5402 case NEON::BI__builtin_neon_vrshrd_n_u64:
5403 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5404 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5405 ? Intrinsic::aarch64_neon_urshl
5406 : Intrinsic::aarch64_neon_srshl;
5408 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
5409 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
5412 case NEON::BI__builtin_neon_vrsrad_n_u64:
5413 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5414 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5415 ? Intrinsic::aarch64_neon_urshl
5416 : Intrinsic::aarch64_neon_srshl;
5423 case NEON::BI__builtin_neon_vshld_n_s64:
5424 case NEON::BI__builtin_neon_vshld_n_u64: {
5427 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
5429 case NEON::BI__builtin_neon_vshrd_n_s64: {
5433 Amt->getZExtValue())),
5436 case NEON::BI__builtin_neon_vshrd_n_u64: {
5438 uint64_t ShiftAmt = Amt->getZExtValue();
5441 return ConstantInt::get(
Int64Ty, 0);
5442 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
5445 case NEON::BI__builtin_neon_vsrad_n_s64: {
5449 Amt->getZExtValue())),
5451 return Builder.CreateAdd(Ops[0], Ops[1]);
5453 case NEON::BI__builtin_neon_vsrad_n_u64: {
5455 uint64_t ShiftAmt = Amt->getZExtValue();
5460 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
5462 return Builder.CreateAdd(Ops[0], Ops[1]);
5464 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
5465 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
5466 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
5467 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
5475 ProductOps,
"vqdmlXl");
5476 Constant *CI = ConstantInt::get(
SizeTy, 0);
5477 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
5480 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
5481 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
5482 ? Intrinsic::aarch64_neon_sqadd
5483 : Intrinsic::aarch64_neon_sqsub;
5486 case NEON::BI__builtin_neon_vqdmlals_s32:
5487 case NEON::BI__builtin_neon_vqdmlsls_s32: {
5489 ProductOps.push_back(Ops[1]);
5493 ProductOps,
"vqdmlXl");
5495 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
5496 ? Intrinsic::aarch64_neon_sqadd
5497 : Intrinsic::aarch64_neon_sqsub;
5500 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
5501 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
5502 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
5503 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
5507 ProductOps.push_back(Ops[1]);
5508 ProductOps.push_back(Ops[2]);
5511 ProductOps,
"vqdmlXl");
5514 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
5515 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
5516 ? Intrinsic::aarch64_neon_sqadd
5517 : Intrinsic::aarch64_neon_sqsub;
5530 AArch64SIMDIntrinsicsProvenSorted);
5534 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
5535 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
5542 switch (BuiltinID) {
5543 default:
return nullptr;
5544 case NEON::BI__builtin_neon_vbsl_v:
5545 case NEON::BI__builtin_neon_vbslq_v: {
5546 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
5551 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
5552 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
5553 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
5556 case NEON::BI__builtin_neon_vfma_lane_v:
5557 case NEON::BI__builtin_neon_vfmaq_lane_v: {
5560 Value *Addend = Ops[0];
5561 Value *Multiplicand = Ops[1];
5562 Value *LaneSource = Ops[2];
5563 Ops[0] = Multiplicand;
5564 Ops[1] = LaneSource;
5568 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
5569 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
5571 llvm::Constant *cst = cast<Constant>(Ops[3]);
5572 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
5574 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
5580 case NEON::BI__builtin_neon_vfma_laneq_v: {
5581 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
5583 if (VTy && VTy->getElementType() ==
DoubleTy) {
5589 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
5591 Value *Result =
Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
5598 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
5599 VTy->getNumElements() * 2);
5601 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
5602 cast<ConstantInt>(Ops[3]));
5603 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
5605 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
5607 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
5614 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
5616 case NEON::BI__builtin_neon_vfmas_lane_f32:
5617 case NEON::BI__builtin_neon_vfmas_laneq_f32:
5618 case NEON::BI__builtin_neon_vfmad_lane_f64:
5619 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
5623 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
5624 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
5626 case NEON::BI__builtin_neon_vmull_v:
5628 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
5629 if (Type.
isPoly()) Int = Intrinsic::aarch64_neon_pmull;
5631 case NEON::BI__builtin_neon_vmax_v:
5632 case NEON::BI__builtin_neon_vmaxq_v:
5634 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
5635 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
5637 case NEON::BI__builtin_neon_vmin_v:
5638 case NEON::BI__builtin_neon_vminq_v:
5640 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
5641 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
5643 case NEON::BI__builtin_neon_vabd_v:
5644 case NEON::BI__builtin_neon_vabdq_v:
5646 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
5647 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
5649 case NEON::BI__builtin_neon_vpadal_v:
5650 case NEON::BI__builtin_neon_vpadalq_v: {
5651 unsigned ArgElts = VTy->getNumElements();
5652 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
5653 unsigned BitWidth = EltTy->getBitWidth();
5655 llvm::IntegerType::get(
getLLVMContext(), BitWidth/2), 2*ArgElts);
5657 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
5659 TmpOps.push_back(Ops[1]);
5663 return Builder.CreateAdd(tmp, addend);
5665 case NEON::BI__builtin_neon_vpmin_v:
5666 case NEON::BI__builtin_neon_vpminq_v:
5668 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
5669 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
5671 case NEON::BI__builtin_neon_vpmax_v:
5672 case NEON::BI__builtin_neon_vpmaxq_v:
5674 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
5675 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
5677 case NEON::BI__builtin_neon_vminnm_v:
5678 case NEON::BI__builtin_neon_vminnmq_v:
5679 Int = Intrinsic::aarch64_neon_fminnm;
5681 case NEON::BI__builtin_neon_vmaxnm_v:
5682 case NEON::BI__builtin_neon_vmaxnmq_v:
5683 Int = Intrinsic::aarch64_neon_fmaxnm;
5685 case NEON::BI__builtin_neon_vrecpss_f32: {
5690 case NEON::BI__builtin_neon_vrecpsd_f64: {
5695 case NEON::BI__builtin_neon_vqshrun_n_v:
5696 Int = Intrinsic::aarch64_neon_sqshrun;
5698 case NEON::BI__builtin_neon_vqrshrun_n_v:
5699 Int = Intrinsic::aarch64_neon_sqrshrun;
5701 case NEON::BI__builtin_neon_vqshrn_n_v:
5702 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
5704 case NEON::BI__builtin_neon_vrshrn_n_v:
5705 Int = Intrinsic::aarch64_neon_rshrn;
5707 case NEON::BI__builtin_neon_vqrshrn_n_v:
5708 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
5710 case NEON::BI__builtin_neon_vrnda_v:
5711 case NEON::BI__builtin_neon_vrndaq_v: {
5715 case NEON::BI__builtin_neon_vrndi_v:
5716 case NEON::BI__builtin_neon_vrndiq_v: {
5720 case NEON::BI__builtin_neon_vrndm_v:
5721 case NEON::BI__builtin_neon_vrndmq_v: {
5725 case NEON::BI__builtin_neon_vrndn_v:
5726 case NEON::BI__builtin_neon_vrndnq_v: {
5727 Int = Intrinsic::aarch64_neon_frintn;
5730 case NEON::BI__builtin_neon_vrndp_v:
5731 case NEON::BI__builtin_neon_vrndpq_v: {
5735 case NEON::BI__builtin_neon_vrndx_v:
5736 case NEON::BI__builtin_neon_vrndxq_v: {
5740 case NEON::BI__builtin_neon_vrnd_v:
5741 case NEON::BI__builtin_neon_vrndq_v: {
5745 case NEON::BI__builtin_neon_vceqz_v:
5746 case NEON::BI__builtin_neon_vceqzq_v:
5748 ICmpInst::ICMP_EQ,
"vceqz");
5749 case NEON::BI__builtin_neon_vcgez_v:
5750 case NEON::BI__builtin_neon_vcgezq_v:
5752 ICmpInst::ICMP_SGE,
"vcgez");
5753 case NEON::BI__builtin_neon_vclez_v:
5754 case NEON::BI__builtin_neon_vclezq_v:
5756 ICmpInst::ICMP_SLE,
"vclez");
5757 case NEON::BI__builtin_neon_vcgtz_v:
5758 case NEON::BI__builtin_neon_vcgtzq_v:
5760 ICmpInst::ICMP_SGT,
"vcgtz");
5761 case NEON::BI__builtin_neon_vcltz_v:
5762 case NEON::BI__builtin_neon_vcltzq_v:
5764 ICmpInst::ICMP_SLT,
"vcltz");
5765 case NEON::BI__builtin_neon_vcvt_f64_v:
5766 case NEON::BI__builtin_neon_vcvtq_f64_v:
5769 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
5770 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
5771 case NEON::BI__builtin_neon_vcvt_f64_f32: {
5773 "unexpected vcvt_f64_f32 builtin");
5777 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
5779 case NEON::BI__builtin_neon_vcvt_f32_f64: {
5781 "unexpected vcvt_f32_f64 builtin");
5785 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
5787 case NEON::BI__builtin_neon_vcvt_s32_v:
5788 case NEON::BI__builtin_neon_vcvt_u32_v:
5789 case NEON::BI__builtin_neon_vcvt_s64_v:
5790 case NEON::BI__builtin_neon_vcvt_u64_v:
5791 case NEON::BI__builtin_neon_vcvtq_s32_v:
5792 case NEON::BI__builtin_neon_vcvtq_u32_v:
5793 case NEON::BI__builtin_neon_vcvtq_s64_v:
5794 case NEON::BI__builtin_neon_vcvtq_u64_v: {
5797 return Builder.CreateFPToUI(Ops[0], Ty);
5798 return Builder.CreateFPToSI(Ops[0], Ty);
5800 case NEON::BI__builtin_neon_vcvta_s32_v:
5801 case NEON::BI__builtin_neon_vcvtaq_s32_v:
5802 case NEON::BI__builtin_neon_vcvta_u32_v:
5803 case NEON::BI__builtin_neon_vcvtaq_u32_v:
5804 case NEON::BI__builtin_neon_vcvta_s64_v:
5805 case NEON::BI__builtin_neon_vcvtaq_s64_v:
5806 case NEON::BI__builtin_neon_vcvta_u64_v:
5807 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
5808 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
5812 case NEON::BI__builtin_neon_vcvtm_s32_v:
5813 case NEON::BI__builtin_neon_vcvtmq_s32_v:
5814 case NEON::BI__builtin_neon_vcvtm_u32_v:
5815 case NEON::BI__builtin_neon_vcvtmq_u32_v:
5816 case NEON::BI__builtin_neon_vcvtm_s64_v:
5817 case NEON::BI__builtin_neon_vcvtmq_s64_v:
5818 case NEON::BI__builtin_neon_vcvtm_u64_v:
5819 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
5820 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
5824 case NEON::BI__builtin_neon_vcvtn_s32_v:
5825 case NEON::BI__builtin_neon_vcvtnq_s32_v:
5826 case NEON::BI__builtin_neon_vcvtn_u32_v:
5827 case NEON::BI__builtin_neon_vcvtnq_u32_v:
5828 case NEON::BI__builtin_neon_vcvtn_s64_v:
5829 case NEON::BI__builtin_neon_vcvtnq_s64_v:
5830 case NEON::BI__builtin_neon_vcvtn_u64_v:
5831 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
5832 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
5836 case NEON::BI__builtin_neon_vcvtp_s32_v:
5837 case NEON::BI__builtin_neon_vcvtpq_s32_v:
5838 case NEON::BI__builtin_neon_vcvtp_u32_v:
5839 case NEON::BI__builtin_neon_vcvtpq_u32_v:
5840 case NEON::BI__builtin_neon_vcvtp_s64_v:
5841 case NEON::BI__builtin_neon_vcvtpq_s64_v:
5842 case NEON::BI__builtin_neon_vcvtp_u64_v:
5843 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
5844 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
5848 case NEON::BI__builtin_neon_vmulx_v:
5849 case NEON::BI__builtin_neon_vmulxq_v: {
5850 Int = Intrinsic::aarch64_neon_fmulx;
5853 case NEON::BI__builtin_neon_vmul_lane_v:
5854 case NEON::BI__builtin_neon_vmul_laneq_v: {
5857 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
5863 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
5864 Value *Result =
Builder.CreateFMul(Ops[0], Ops[1]);
5867 case NEON::BI__builtin_neon_vnegd_s64:
5869 case NEON::BI__builtin_neon_vpmaxnm_v:
5870 case NEON::BI__builtin_neon_vpmaxnmq_v: {
5871 Int = Intrinsic::aarch64_neon_fmaxnmp;
5874 case NEON::BI__builtin_neon_vpminnm_v:
5875 case NEON::BI__builtin_neon_vpminnmq_v: {
5876 Int = Intrinsic::aarch64_neon_fminnmp;
5879 case NEON::BI__builtin_neon_vsqrt_v:
5880 case NEON::BI__builtin_neon_vsqrtq_v: {
5885 case NEON::BI__builtin_neon_vrbit_v:
5886 case NEON::BI__builtin_neon_vrbitq_v: {
5887 Int = Intrinsic::aarch64_neon_rbit;
5890 case NEON::BI__builtin_neon_vaddv_u8:
5894 case NEON::BI__builtin_neon_vaddv_s8: {
5895 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
5897 VTy = llvm::VectorType::get(
Int8Ty, 8);
5903 case NEON::BI__builtin_neon_vaddv_u16:
5906 case NEON::BI__builtin_neon_vaddv_s16: {
5907 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
5909 VTy = llvm::VectorType::get(
Int16Ty, 4);
5915 case NEON::BI__builtin_neon_vaddvq_u8:
5918 case NEON::BI__builtin_neon_vaddvq_s8: {
5919 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
5921 VTy = llvm::VectorType::get(
Int8Ty, 16);
5927 case NEON::BI__builtin_neon_vaddvq_u16:
5930 case NEON::BI__builtin_neon_vaddvq_s16: {
5931 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
5933 VTy = llvm::VectorType::get(
Int16Ty, 8);
5939 case NEON::BI__builtin_neon_vmaxv_u8: {
5940 Int = Intrinsic::aarch64_neon_umaxv;
5942 VTy = llvm::VectorType::get(
Int8Ty, 8);
5948 case NEON::BI__builtin_neon_vmaxv_u16: {
5949 Int = Intrinsic::aarch64_neon_umaxv;
5951 VTy = llvm::VectorType::get(
Int16Ty, 4);
5957 case NEON::BI__builtin_neon_vmaxvq_u8: {
5958 Int = Intrinsic::aarch64_neon_umaxv;
5960 VTy = llvm::VectorType::get(
Int8Ty, 16);
5966 case NEON::BI__builtin_neon_vmaxvq_u16: {
5967 Int = Intrinsic::aarch64_neon_umaxv;
5969 VTy = llvm::VectorType::get(
Int16Ty, 8);
5975 case NEON::BI__builtin_neon_vmaxv_s8: {
5976 Int = Intrinsic::aarch64_neon_smaxv;
5978 VTy = llvm::VectorType::get(
Int8Ty, 8);
5984 case NEON::BI__builtin_neon_vmaxv_s16: {
5985 Int = Intrinsic::aarch64_neon_smaxv;
5987 VTy = llvm::VectorType::get(
Int16Ty, 4);
5993 case NEON::BI__builtin_neon_vmaxvq_s8: {
5994 Int = Intrinsic::aarch64_neon_smaxv;
5996 VTy = llvm::VectorType::get(
Int8Ty, 16);
6002 case NEON::BI__builtin_neon_vmaxvq_s16: {
6003 Int = Intrinsic::aarch64_neon_smaxv;
6005 VTy = llvm::VectorType::get(
Int16Ty, 8);
6011 case NEON::BI__builtin_neon_vminv_u8: {
6012 Int = Intrinsic::aarch64_neon_uminv;
6014 VTy = llvm::VectorType::get(
Int8Ty, 8);
6020 case NEON::BI__builtin_neon_vminv_u16: {
6021 Int = Intrinsic::aarch64_neon_uminv;
6023 VTy = llvm::VectorType::get(
Int16Ty, 4);
6029 case NEON::BI__builtin_neon_vminvq_u8: {
6030 Int = Intrinsic::aarch64_neon_uminv;
6032 VTy = llvm::VectorType::get(
Int8Ty, 16);
6038 case NEON::BI__builtin_neon_vminvq_u16: {
6039 Int = Intrinsic::aarch64_neon_uminv;
6041 VTy = llvm::VectorType::get(
Int16Ty, 8);
6047 case NEON::BI__builtin_neon_vminv_s8: {
6048 Int = Intrinsic::aarch64_neon_sminv;
6050 VTy = llvm::VectorType::get(
Int8Ty, 8);
6056 case NEON::BI__builtin_neon_vminv_s16: {
6057 Int = Intrinsic::aarch64_neon_sminv;
6059 VTy = llvm::VectorType::get(
Int16Ty, 4);
6065 case NEON::BI__builtin_neon_vminvq_s8: {
6066 Int = Intrinsic::aarch64_neon_sminv;
6068 VTy = llvm::VectorType::get(
Int8Ty, 16);
6074 case NEON::BI__builtin_neon_vminvq_s16: {
6075 Int = Intrinsic::aarch64_neon_sminv;
6077 VTy = llvm::VectorType::get(
Int16Ty, 8);
6083 case NEON::BI__builtin_neon_vmul_n_f64: {
6086 return Builder.CreateFMul(Ops[0], RHS);
6088 case NEON::BI__builtin_neon_vaddlv_u8: {
6089 Int = Intrinsic::aarch64_neon_uaddlv;
6091 VTy = llvm::VectorType::get(
Int8Ty, 8);
6097 case NEON::BI__builtin_neon_vaddlv_u16: {
6098 Int = Intrinsic::aarch64_neon_uaddlv;
6100 VTy = llvm::VectorType::get(
Int16Ty, 4);
6105 case NEON::BI__builtin_neon_vaddlvq_u8: {
6106 Int = Intrinsic::aarch64_neon_uaddlv;
6108 VTy = llvm::VectorType::get(
Int8Ty, 16);
6114 case NEON::BI__builtin_neon_vaddlvq_u16: {
6115 Int = Intrinsic::aarch64_neon_uaddlv;
6117 VTy = llvm::VectorType::get(
Int16Ty, 8);
6122 case NEON::BI__builtin_neon_vaddlv_s8: {
6123 Int = Intrinsic::aarch64_neon_saddlv;
6125 VTy = llvm::VectorType::get(
Int8Ty, 8);
6131 case NEON::BI__builtin_neon_vaddlv_s16: {
6132 Int = Intrinsic::aarch64_neon_saddlv;
6134 VTy = llvm::VectorType::get(
Int16Ty, 4);
6139 case NEON::BI__builtin_neon_vaddlvq_s8: {
6140 Int = Intrinsic::aarch64_neon_saddlv;
6142 VTy = llvm::VectorType::get(
Int8Ty, 16);
6148 case NEON::BI__builtin_neon_vaddlvq_s16: {
6149 Int = Intrinsic::aarch64_neon_saddlv;
6151 VTy = llvm::VectorType::get(
Int16Ty, 8);
6156 case NEON::BI__builtin_neon_vsri_n_v:
6157 case NEON::BI__builtin_neon_vsriq_n_v: {
6158 Int = Intrinsic::aarch64_neon_vsri;
6162 case NEON::BI__builtin_neon_vsli_n_v:
6163 case NEON::BI__builtin_neon_vsliq_n_v: {
6164 Int = Intrinsic::aarch64_neon_vsli;
6168 case NEON::BI__builtin_neon_vsra_n_v:
6169 case NEON::BI__builtin_neon_vsraq_n_v:
6172 return Builder.CreateAdd(Ops[0], Ops[1]);
6173 case NEON::BI__builtin_neon_vrsra_n_v:
6174 case NEON::BI__builtin_neon_vrsraq_n_v: {
6175 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6177 TmpOps.push_back(Ops[1]);
6178 TmpOps.push_back(Ops[2]);
6182 return Builder.CreateAdd(Ops[0], tmp);
6186 case NEON::BI__builtin_neon_vld1_x2_v:
6187 case NEON::BI__builtin_neon_vld1q_x2_v:
6188 case NEON::BI__builtin_neon_vld1_x3_v:
6189 case NEON::BI__builtin_neon_vld1q_x3_v:
6190 case NEON::BI__builtin_neon_vld1_x4_v:
6191 case NEON::BI__builtin_neon_vld1q_x4_v: {
6192 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6196 switch (BuiltinID) {
6197 case NEON::BI__builtin_neon_vld1_x2_v:
6198 case NEON::BI__builtin_neon_vld1q_x2_v:
6199 Int = Intrinsic::aarch64_neon_ld1x2;
6201 case NEON::BI__builtin_neon_vld1_x3_v:
6202 case NEON::BI__builtin_neon_vld1q_x3_v:
6203 Int = Intrinsic::aarch64_neon_ld1x3;
6205 case NEON::BI__builtin_neon_vld1_x4_v:
6206 case NEON::BI__builtin_neon_vld1q_x4_v:
6207 Int = Intrinsic::aarch64_neon_ld1x4;
6211 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
6212 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6216 case NEON::BI__builtin_neon_vst1_x2_v:
6217 case NEON::BI__builtin_neon_vst1q_x2_v:
6218 case NEON::BI__builtin_neon_vst1_x3_v:
6219 case NEON::BI__builtin_neon_vst1q_x3_v:
6220 case NEON::BI__builtin_neon_vst1_x4_v:
6221 case NEON::BI__builtin_neon_vst1q_x4_v: {
6222 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6225 switch (BuiltinID) {
6226 case NEON::BI__builtin_neon_vst1_x2_v:
6227 case NEON::BI__builtin_neon_vst1q_x2_v:
6228 Int = Intrinsic::aarch64_neon_st1x2;
6230 case NEON::BI__builtin_neon_vst1_x3_v:
6231 case NEON::BI__builtin_neon_vst1q_x3_v:
6232 Int = Intrinsic::aarch64_neon_st1x3;
6234 case NEON::BI__builtin_neon_vst1_x4_v:
6235 case NEON::BI__builtin_neon_vst1q_x4_v:
6236 Int = Intrinsic::aarch64_neon_st1x4;
6239 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6242 case NEON::BI__builtin_neon_vld1_v:
6243 case NEON::BI__builtin_neon_vld1q_v:
6246 case NEON::BI__builtin_neon_vst1_v:
6247 case NEON::BI__builtin_neon_vst1q_v:
6251 case NEON::BI__builtin_neon_vld1_lane_v:
6252 case NEON::BI__builtin_neon_vld1q_lane_v:
6254 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6257 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
6258 case NEON::BI__builtin_neon_vld1_dup_v:
6259 case NEON::BI__builtin_neon_vld1q_dup_v: {
6260 Value *V = UndefValue::get(Ty);
6261 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6264 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
6265 Ops[0] =
Builder.CreateInsertElement(V, Ops[0], CI);
6268 case NEON::BI__builtin_neon_vst1_lane_v:
6269 case NEON::BI__builtin_neon_vst1q_lane_v:
6271 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
6272 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6275 case NEON::BI__builtin_neon_vld2_v:
6276 case NEON::BI__builtin_neon_vld2q_v: {
6277 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6281 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6283 llvm::PointerType::getUnqual(Ops[1]->getType()));
6286 case NEON::BI__builtin_neon_vld3_v:
6287 case NEON::BI__builtin_neon_vld3q_v: {
6288 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6292 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6294 llvm::PointerType::getUnqual(Ops[1]->getType()));
6297 case NEON::BI__builtin_neon_vld4_v:
6298 case NEON::BI__builtin_neon_vld4q_v: {
6299 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6303 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6305 llvm::PointerType::getUnqual(Ops[1]->getType()));
6308 case NEON::BI__builtin_neon_vld2_dup_v:
6309 case NEON::BI__builtin_neon_vld2q_dup_v: {
6311 llvm::PointerType::getUnqual(VTy->getElementType());
6315 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6317 llvm::PointerType::getUnqual(Ops[1]->getType()));
6320 case NEON::BI__builtin_neon_vld3_dup_v:
6321 case NEON::BI__builtin_neon_vld3q_dup_v: {
6323 llvm::PointerType::getUnqual(VTy->getElementType());
6327 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6329 llvm::PointerType::getUnqual(Ops[1]->getType()));
6332 case NEON::BI__builtin_neon_vld4_dup_v:
6333 case NEON::BI__builtin_neon_vld4q_dup_v: {
6335 llvm::PointerType::getUnqual(VTy->getElementType());
6339 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6341 llvm::PointerType::getUnqual(Ops[1]->getType()));
6344 case NEON::BI__builtin_neon_vld2_lane_v:
6345 case NEON::BI__builtin_neon_vld2q_lane_v: {
6346 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6348 Ops.push_back(Ops[1]);
6349 Ops.erase(Ops.begin()+1);
6353 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld2_lane");
6354 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6358 case NEON::BI__builtin_neon_vld3_lane_v:
6359 case NEON::BI__builtin_neon_vld3q_lane_v: {
6360 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6362 Ops.push_back(Ops[1]);
6363 Ops.erase(Ops.begin()+1);
6368 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld3_lane");
6369 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6373 case NEON::BI__builtin_neon_vld4_lane_v:
6374 case NEON::BI__builtin_neon_vld4q_lane_v: {
6375 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6377 Ops.push_back(Ops[1]);
6378 Ops.erase(Ops.begin()+1);
6384 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld4_lane");
6385 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6389 case NEON::BI__builtin_neon_vst2_v:
6390 case NEON::BI__builtin_neon_vst2q_v: {
6391 Ops.push_back(Ops[0]);
6392 Ops.erase(Ops.begin());
6393 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6397 case NEON::BI__builtin_neon_vst2_lane_v:
6398 case NEON::BI__builtin_neon_vst2q_lane_v: {
6399 Ops.push_back(Ops[0]);
6400 Ops.erase(Ops.begin());
6402 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6406 case NEON::BI__builtin_neon_vst3_v:
6407 case NEON::BI__builtin_neon_vst3q_v: {
6408 Ops.push_back(Ops[0]);
6409 Ops.erase(Ops.begin());
6410 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6414 case NEON::BI__builtin_neon_vst3_lane_v:
6415 case NEON::BI__builtin_neon_vst3q_lane_v: {
6416 Ops.push_back(Ops[0]);
6417 Ops.erase(Ops.begin());
6419 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6423 case NEON::BI__builtin_neon_vst4_v:
6424 case NEON::BI__builtin_neon_vst4q_v: {
6425 Ops.push_back(Ops[0]);
6426 Ops.erase(Ops.begin());
6427 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6431 case NEON::BI__builtin_neon_vst4_lane_v:
6432 case NEON::BI__builtin_neon_vst4q_lane_v: {
6433 Ops.push_back(Ops[0]);
6434 Ops.erase(Ops.begin());
6436 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6440 case NEON::BI__builtin_neon_vtrn_v:
6441 case NEON::BI__builtin_neon_vtrnq_v: {
6445 Value *SV =
nullptr;
6447 for (
unsigned vi = 0; vi != 2; ++vi) {
6449 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6450 Indices.push_back(i+vi);
6451 Indices.push_back(i+e+vi);
6453 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6454 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
6459 case NEON::BI__builtin_neon_vuzp_v:
6460 case NEON::BI__builtin_neon_vuzpq_v: {
6464 Value *SV =
nullptr;
6466 for (
unsigned vi = 0; vi != 2; ++vi) {
6468 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6469 Indices.push_back(2*i+vi);
6471 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6472 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
6477 case NEON::BI__builtin_neon_vzip_v:
6478 case NEON::BI__builtin_neon_vzipq_v: {
6482 Value *SV =
nullptr;
6484 for (
unsigned vi = 0; vi != 2; ++vi) {
6486 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6487 Indices.push_back((i + vi*e) >> 1);
6488 Indices.push_back(((i + vi*e) >> 1)+e);
6490 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6491 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
6496 case NEON::BI__builtin_neon_vqtbl1q_v: {
6500 case NEON::BI__builtin_neon_vqtbl2q_v: {
6504 case NEON::BI__builtin_neon_vqtbl3q_v: {
6508 case NEON::BI__builtin_neon_vqtbl4q_v: {
6512 case NEON::BI__builtin_neon_vqtbx1q_v: {
6516 case NEON::BI__builtin_neon_vqtbx2q_v: {
6520 case NEON::BI__builtin_neon_vqtbx3q_v: {
6524 case NEON::BI__builtin_neon_vqtbx4q_v: {
6528 case NEON::BI__builtin_neon_vsqadd_v:
6529 case NEON::BI__builtin_neon_vsqaddq_v: {
6530 Int = Intrinsic::aarch64_neon_usqadd;
6533 case NEON::BI__builtin_neon_vuqadd_v:
6534 case NEON::BI__builtin_neon_vuqaddq_v: {
6535 Int = Intrinsic::aarch64_neon_suqadd;
6543 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
6544 "Not a power-of-two sized vector!");
6545 bool AllConstants =
true;
6546 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
6547 AllConstants &= isa<Constant>(Ops[i]);
6552 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
6553 CstOps.push_back(cast<Constant>(Ops[i]));
6554 return llvm::ConstantVector::get(CstOps);
6559 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
6561 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
6562 Result =
Builder.CreateInsertElement(Result, Ops[i],
Builder.getInt32(i));
6571 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
6572 cast<IntegerType>(Mask->getType())->getBitWidth());
6578 uint32_t Indices[4];
6579 for (
unsigned i = 0; i != NumElts; ++i)
6581 MaskVec = CGF.
Builder.CreateShuffleVector(MaskVec, MaskVec,
6582 makeArrayRef(Indices, NumElts),
6593 llvm::PointerType::getUnqual(Ops[1]->getType()));
6596 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
6597 if (C->isAllOnesValue())
6601 Ops[1]->getType()->getVectorNumElements());
6603 return CGF.
Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec);
6610 llvm::PointerType::getUnqual(Ops[1]->getType()));
6613 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
6614 if (C->isAllOnesValue())
6618 Ops[1]->getType()->getVectorNumElements());
6620 return CGF.
Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]);
6624 Value *Mask, Value *Op0, Value *Op1) {
6627 if (
const auto *C = dyn_cast<Constant>(Mask))
6628 if (C->isAllOnesValue())
6631 Mask =
getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
6633 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
6638 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
6642 Cmp = Constant::getNullValue(
6643 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
6644 }
else if (CC == 7) {
6645 Cmp = Constant::getAllOnesValue(
6646 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
6648 ICmpInst::Predicate Pred;
6650 default: llvm_unreachable(
"Unknown condition code");
6651 case 0: Pred = ICmpInst::ICMP_EQ;
break;
6652 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
6653 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
6654 case 4: Pred = ICmpInst::ICMP_NE;
break;
6655 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
6656 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
6658 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
6661 const auto *C = dyn_cast<Constant>(Ops.back());
6662 if (!C || !C->isAllOnesValue())
6666 uint32_t Indices[8];
6667 for (
unsigned i = 0; i != NumElts; ++i)
6669 for (
unsigned i = NumElts; i != 8; ++i)
6670 Indices[i] = i % NumElts + NumElts;
6671 Cmp = CGF.
Builder.CreateShuffleVector(
6672 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
6681 if (BuiltinID == X86::BI__builtin_ms_va_start ||
6682 BuiltinID == X86::BI__builtin_ms_va_end)
6684 BuiltinID == X86::BI__builtin_ms_va_start);
6685 if (BuiltinID == X86::BI__builtin_ms_va_copy) {
6709 unsigned ICEArguments = 0;
6714 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
6716 if ((ICEArguments & (1 << i)) == 0) {
6723 llvm::APSInt Result;
6725 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
6735 auto getCmpIntrinsicCall = [
this, &Ops](
Intrinsic::ID ID,
unsigned Imm) {
6736 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
6738 return Builder.CreateCall(F, Ops);
6746 auto getVectorFCmpIR = [
this, &Ops](CmpInst::Predicate Pred) {
6747 Value *Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
6748 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
6749 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
6750 Value *Sext =
Builder.CreateSExt(Cmp, IntVecTy);
6754 switch (BuiltinID) {
6755 default:
return nullptr;
6756 case X86::BI__builtin_cpu_supports: {
6758 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
6795 X86Features Feature = StringSwitch<X86Features>(FeatureStr)
6796 .Case(
"cmov", X86Features::CMOV)
6797 .Case(
"mmx", X86Features::MMX)
6798 .Case(
"popcnt", X86Features::POPCNT)
6799 .Case(
"sse", X86Features::SSE)
6800 .Case(
"sse2", X86Features::SSE2)
6801 .Case(
"sse3", X86Features::SSE3)
6802 .Case(
"ssse3", X86Features::SSSE3)
6803 .Case(
"sse4.1", X86Features::SSE4_1)
6804 .Case(
"sse4.2", X86Features::SSE4_2)
6805 .Case(
"avx", X86Features::AVX)
6806 .Case(
"avx2", X86Features::AVX2)
6807 .Case(
"sse4a", X86Features::SSE4_A)
6808 .Case(
"fma4", X86Features::FMA4)
6809 .Case(
"xop", X86Features::XOP)
6810 .Case(
"fma", X86Features::FMA)
6811 .Case(
"avx512f", X86Features::AVX512F)
6812 .Case(
"bmi", X86Features::BMI)
6813 .Case(
"bmi2", X86Features::BMI2)
6814 .Case(
"aes", X86Features::AES)
6815 .Case(
"pclmul", X86Features::PCLMUL)
6816 .Case(
"avx512vl", X86Features::AVX512VL)
6817 .Case(
"avx512bw", X86Features::AVX512BW)
6818 .Case(
"avx512dq", X86Features::AVX512DQ)
6819 .Case(
"avx512cd", X86Features::AVX512CD)
6820 .Case(
"avx512er", X86Features::AVX512ER)
6821 .Case(
"avx512pf", X86Features::AVX512PF)
6822 .Case(
"avx512vbmi", X86Features::AVX512VBMI)
6823 .Case(
"avx512ifma", X86Features::AVX512IFMA)
6824 .Default(X86Features::MAX);
6825 assert(Feature != X86Features::MAX &&
"Invalid feature!");
6846 Value *CpuFeatures =
Builder.CreateGEP(STy, CpuModel, Idxs);
6851 Value *Bitset =
Builder.CreateAnd(
6852 Features, llvm::ConstantInt::get(
Int32Ty, 1ULL << Feature));
6853 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(
Int32Ty, 0));
6855 case X86::BI_mm_prefetch: {
6856 Value *Address = Ops[0];
6857 Value *RW = ConstantInt::get(
Int32Ty, 0);
6858 Value *Locality = Ops[1];
6859 Value *Data = ConstantInt::get(
Int32Ty, 1);
6861 return Builder.CreateCall(F, {Address, RW, Locality, Data});
6863 case X86::BI__builtin_ia32_undef128:
6864 case X86::BI__builtin_ia32_undef256:
6865 case X86::BI__builtin_ia32_undef512:
6867 case X86::BI__builtin_ia32_vec_init_v8qi:
6868 case X86::BI__builtin_ia32_vec_init_v4hi:
6869 case X86::BI__builtin_ia32_vec_init_v2si:
6872 case X86::BI__builtin_ia32_vec_ext_v2si:
6873 return Builder.CreateExtractElement(Ops[0],
6874 llvm::ConstantInt::get(Ops[1]->getType(), 0));
6875 case X86::BI__builtin_ia32_ldmxcsr: {
6881 case X86::BI__builtin_ia32_stmxcsr: {
6887 case X86::BI__builtin_ia32_xsave:
6888 case X86::BI__builtin_ia32_xsave64:
6889 case X86::BI__builtin_ia32_xrstor:
6890 case X86::BI__builtin_ia32_xrstor64:
6891 case X86::BI__builtin_ia32_xsaveopt:
6892 case X86::BI__builtin_ia32_xsaveopt64:
6893 case X86::BI__builtin_ia32_xrstors:
6894 case X86::BI__builtin_ia32_xrstors64:
6895 case X86::BI__builtin_ia32_xsavec:
6896 case X86::BI__builtin_ia32_xsavec64:
6897 case X86::BI__builtin_ia32_xsaves:
6898 case X86::BI__builtin_ia32_xsaves64: {
6900 #define INTRINSIC_X86_XSAVE_ID(NAME) \
6901 case X86::BI__builtin_ia32_##NAME: \
6902 ID = Intrinsic::x86_##NAME; \
6904 switch (BuiltinID) {
6905 default: llvm_unreachable(
"Unsupported intrinsic!");
6919 #undef INTRINSIC_X86_XSAVE_ID
6920 Value *Mhi =
Builder.CreateTrunc(
6927 case X86::BI__builtin_ia32_storedqudi128_mask:
6928 case X86::BI__builtin_ia32_storedqusi128_mask:
6929 case X86::BI__builtin_ia32_storedquhi128_mask:
6930 case X86::BI__builtin_ia32_storedquqi128_mask:
6931 case X86::BI__builtin_ia32_storeupd128_mask:
6932 case X86::BI__builtin_ia32_storeups128_mask:
6933 case X86::BI__builtin_ia32_storedqudi256_mask:
6934 case X86::BI__builtin_ia32_storedqusi256_mask:
6935 case X86::BI__builtin_ia32_storedquhi256_mask:
6936 case X86::BI__builtin_ia32_storedquqi256_mask:
6937 case X86::BI__builtin_ia32_storeupd256_mask:
6938 case X86::BI__builtin_ia32_storeups256_mask:
6939 case X86::BI__builtin_ia32_storedqudi512_mask:
6940 case X86::BI__builtin_ia32_storedqusi512_mask:
6941 case X86::BI__builtin_ia32_storedquhi512_mask:
6942 case X86::BI__builtin_ia32_storedquqi512_mask:
6943 case X86::BI__builtin_ia32_storeupd512_mask:
6944 case X86::BI__builtin_ia32_storeups512_mask:
6947 case X86::BI__builtin_ia32_movdqa32store128_mask:
6948 case X86::BI__builtin_ia32_movdqa64store128_mask:
6949 case X86::BI__builtin_ia32_storeaps128_mask:
6950 case X86::BI__builtin_ia32_storeapd128_mask:
6951 case X86::BI__builtin_ia32_movdqa32store256_mask:
6952 case X86::BI__builtin_ia32_movdqa64store256_mask:
6953 case X86::BI__builtin_ia32_storeaps256_mask:
6954 case X86::BI__builtin_ia32_storeapd256_mask:
6955 case X86::BI__builtin_ia32_movdqa32store512_mask:
6956 case X86::BI__builtin_ia32_movdqa64store512_mask:
6957 case X86::BI__builtin_ia32_storeaps512_mask:
6958 case X86::BI__builtin_ia32_storeapd512_mask: {
6963 case X86::BI__builtin_ia32_loadups128_mask:
6964 case X86::BI__builtin_ia32_loadups256_mask:
6965 case X86::BI__builtin_ia32_loadups512_mask:
6966 case X86::BI__builtin_ia32_loadupd128_mask:
6967 case X86::BI__builtin_ia32_loadupd256_mask:
6968 case X86::BI__builtin_ia32_loadupd512_mask:
6969 case X86::BI__builtin_ia32_loaddquqi128_mask:
6970 case X86::BI__builtin_ia32_loaddquqi256_mask:
6971 case X86::BI__builtin_ia32_loaddquqi512_mask:
6972 case X86::BI__builtin_ia32_loaddquhi128_mask:
6973 case X86::BI__builtin_ia32_loaddquhi256_mask:
6974 case X86::BI__builtin_ia32_loaddquhi512_mask:
6975 case X86::BI__builtin_ia32_loaddqusi128_mask:
6976 case X86::BI__builtin_ia32_loaddqusi256_mask:
6977 case X86::BI__builtin_ia32_loaddqusi512_mask:
6978 case X86::BI__builtin_ia32_loaddqudi128_mask:
6979 case X86::BI__builtin_ia32_loaddqudi256_mask:
6980 case X86::BI__builtin_ia32_loaddqudi512_mask:
6983 case X86::BI__builtin_ia32_loadaps128_mask:
6984 case X86::BI__builtin_ia32_loadaps256_mask:
6985 case X86::BI__builtin_ia32_loadaps512_mask:
6986 case X86::BI__builtin_ia32_loadapd128_mask:
6987 case X86::BI__builtin_ia32_loadapd256_mask:
6988 case X86::BI__builtin_ia32_loadapd512_mask:
6989 case X86::BI__builtin_ia32_movdqa32load128_mask:
6990 case X86::BI__builtin_ia32_movdqa32load256_mask:
6991 case X86::BI__builtin_ia32_movdqa32load512_mask:
6992 case X86::BI__builtin_ia32_movdqa64load128_mask:
6993 case X86::BI__builtin_ia32_movdqa64load256_mask:
6994 case X86::BI__builtin_ia32_movdqa64load512_mask: {
6999 case X86::BI__builtin_ia32_storehps:
7000 case X86::BI__builtin_ia32_storelps: {
7008 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
7010 Ops[1] =
Builder.CreateExtractElement(Ops[1], Idx,
"extract");
7016 case X86::BI__builtin_ia32_palignr128:
7017 case X86::BI__builtin_ia32_palignr256:
7018 case X86::BI__builtin_ia32_palignr128_mask:
7019 case X86::BI__builtin_ia32_palignr256_mask:
7020 case X86::BI__builtin_ia32_palignr512_mask: {
7021 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7023 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
7024 assert(NumElts % 16 == 0);
7033 if (ShiftVal > 16) {
7036 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
7039 uint32_t Indices[64];
7041 for (
unsigned l = 0; l != NumElts; l += 16) {
7042 for (
unsigned i = 0; i != 16; ++i) {
7043 unsigned Idx = ShiftVal + i;
7045 Idx += NumElts - 16;
7046 Indices[l + i] = Idx + l;
7050 Value *Align =
Builder.CreateShuffleVector(Ops[1], Ops[0],
7051 makeArrayRef(Indices, NumElts),
7055 if (Ops.size() == 3)
7061 case X86::BI__builtin_ia32_movnti:
7062 case X86::BI__builtin_ia32_movnti64: {
7063 llvm::MDNode *
Node = llvm::MDNode::get(
7068 llvm::PointerType::getUnqual(Ops[1]->getType()),
7074 SI->setAlignment(1);
7077 case X86::BI__builtin_ia32_movntsd:
7078 case X86::BI__builtin_ia32_movntss: {
7079 llvm::MDNode *
Node = llvm::MDNode::get(
7083 Value *Scl =
Builder.CreateExtractElement(Ops[1], (uint64_t)0,
"extract");
7087 llvm::PointerType::getUnqual(Scl->getType()),
7093 SI->setAlignment(1);
7097 case X86::BI__builtin_ia32_selectb_128:
7098 case X86::BI__builtin_ia32_selectb_256:
7099 case X86::BI__builtin_ia32_selectb_512:
7100 case X86::BI__builtin_ia32_selectw_128:
7101 case X86::BI__builtin_ia32_selectw_256:
7102 case X86::BI__builtin_ia32_selectw_512:
7103 case X86::BI__builtin_ia32_selectd_128:
7104 case X86::BI__builtin_ia32_selectd_256:
7105 case X86::BI__builtin_ia32_selectd_512:
7106 case X86::BI__builtin_ia32_selectq_128:
7107 case X86::BI__builtin_ia32_selectq_256:
7108 case X86::BI__builtin_ia32_selectq_512:
7109 case X86::BI__builtin_ia32_selectps_128:
7110 case X86::BI__builtin_ia32_selectps_256:
7111 case X86::BI__builtin_ia32_selectps_512:
7112 case X86::BI__builtin_ia32_selectpd_128:
7113 case X86::BI__builtin_ia32_selectpd_256:
7114 case X86::BI__builtin_ia32_selectpd_512:
7116 case X86::BI__builtin_ia32_pcmpeqb128_mask:
7117 case X86::BI__builtin_ia32_pcmpeqb256_mask:
7118 case X86::BI__builtin_ia32_pcmpeqb512_mask:
7119 case X86::BI__builtin_ia32_pcmpeqw128_mask:
7120 case X86::BI__builtin_ia32_pcmpeqw256_mask:
7121 case X86::BI__builtin_ia32_pcmpeqw512_mask:
7122 case X86::BI__builtin_ia32_pcmpeqd128_mask:
7123 case X86::BI__builtin_ia32_pcmpeqd256_mask:
7124 case X86::BI__builtin_ia32_pcmpeqd512_mask:
7125 case X86::BI__builtin_ia32_pcmpeqq128_mask:
7126 case X86::BI__builtin_ia32_pcmpeqq256_mask:
7127 case X86::BI__builtin_ia32_pcmpeqq512_mask:
7129 case X86::BI__builtin_ia32_pcmpgtb128_mask:
7130 case X86::BI__builtin_ia32_pcmpgtb256_mask:
7131 case X86::BI__builtin_ia32_pcmpgtb512_mask:
7132 case X86::BI__builtin_ia32_pcmpgtw128_mask:
7133 case X86::BI__builtin_ia32_pcmpgtw256_mask:
7134 case X86::BI__builtin_ia32_pcmpgtw512_mask:
7135 case X86::BI__builtin_ia32_pcmpgtd128_mask:
7136 case X86::BI__builtin_ia32_pcmpgtd256_mask:
7137 case X86::BI__builtin_ia32_pcmpgtd512_mask:
7138 case X86::BI__builtin_ia32_pcmpgtq128_mask:
7139 case X86::BI__builtin_ia32_pcmpgtq256_mask:
7140 case X86::BI__builtin_ia32_pcmpgtq512_mask:
7142 case X86::BI__builtin_ia32_cmpb128_mask:
7143 case X86::BI__builtin_ia32_cmpb256_mask:
7144 case X86::BI__builtin_ia32_cmpb512_mask:
7145 case X86::BI__builtin_ia32_cmpw128_mask:
7146 case X86::BI__builtin_ia32_cmpw256_mask:
7147 case X86::BI__builtin_ia32_cmpw512_mask:
7148 case X86::BI__builtin_ia32_cmpd128_mask:
7149 case X86::BI__builtin_ia32_cmpd256_mask:
7150 case X86::BI__builtin_ia32_cmpd512_mask:
7151 case X86::BI__builtin_ia32_cmpq128_mask:
7152 case X86::BI__builtin_ia32_cmpq256_mask:
7153 case X86::BI__builtin_ia32_cmpq512_mask: {
7154 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7157 case X86::BI__builtin_ia32_ucmpb128_mask:
7158 case X86::BI__builtin_ia32_ucmpb256_mask:
7159 case X86::BI__builtin_ia32_ucmpb512_mask:
7160 case X86::BI__builtin_ia32_ucmpw128_mask:
7161 case X86::BI__builtin_ia32_ucmpw256_mask:
7162 case X86::BI__builtin_ia32_ucmpw512_mask:
7163 case X86::BI__builtin_ia32_ucmpd128_mask:
7164 case X86::BI__builtin_ia32_ucmpd256_mask:
7165 case X86::BI__builtin_ia32_ucmpd512_mask:
7166 case X86::BI__builtin_ia32_ucmpq128_mask:
7167 case X86::BI__builtin_ia32_ucmpq256_mask:
7168 case X86::BI__builtin_ia32_ucmpq512_mask: {
7169 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7173 case X86::BI__builtin_ia32_vplzcntd_128_mask:
7174 case X86::BI__builtin_ia32_vplzcntd_256_mask:
7175 case X86::BI__builtin_ia32_vplzcntd_512_mask:
7176 case X86::BI__builtin_ia32_vplzcntq_128_mask:
7177 case X86::BI__builtin_ia32_vplzcntq_256_mask:
7178 case X86::BI__builtin_ia32_vplzcntq_512_mask: {
7186 case X86::BI__builtin_ia32_pmaxsb128:
7187 case X86::BI__builtin_ia32_pmaxsw128:
7188 case X86::BI__builtin_ia32_pmaxsd128:
7189 case X86::BI__builtin_ia32_pmaxsb256:
7190 case X86::BI__builtin_ia32_pmaxsw256:
7191 case X86::BI__builtin_ia32_pmaxsd256: {
7192 Value *Cmp =
Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Ops[1]);
7193 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7195 case X86::BI__builtin_ia32_pmaxub128:
7196 case X86::BI__builtin_ia32_pmaxuw128:
7197 case X86::BI__builtin_ia32_pmaxud128:
7198 case X86::BI__builtin_ia32_pmaxub256:
7199 case X86::BI__builtin_ia32_pmaxuw256:
7200 case X86::BI__builtin_ia32_pmaxud256: {
7201 Value *Cmp =
Builder.CreateICmp(ICmpInst::ICMP_UGT, Ops[0], Ops[1]);
7202 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7204 case X86::BI__builtin_ia32_pminsb128:
7205 case X86::BI__builtin_ia32_pminsw128:
7206 case X86::BI__builtin_ia32_pminsd128:
7207 case X86::BI__builtin_ia32_pminsb256:
7208 case X86::BI__builtin_ia32_pminsw256:
7209 case X86::BI__builtin_ia32_pminsd256: {
7210 Value *Cmp =
Builder.CreateICmp(ICmpInst::ICMP_SLT, Ops[0], Ops[1]);
7211 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7213 case X86::BI__builtin_ia32_pminub128:
7214 case X86::BI__builtin_ia32_pminuw128:
7215 case X86::BI__builtin_ia32_pminud128:
7216 case X86::BI__builtin_ia32_pminub256:
7217 case X86::BI__builtin_ia32_pminuw256:
7218 case X86::BI__builtin_ia32_pminud256: {
7219 Value *Cmp =
Builder.CreateICmp(ICmpInst::ICMP_ULT, Ops[0], Ops[1]);
7220 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7224 case X86::BI__builtin_ia32_pswapdsf:
7225 case X86::BI__builtin_ia32_pswapdsi: {
7229 return Builder.CreateCall(F, Ops,
"pswapd");
7231 case X86::BI__builtin_ia32_rdrand16_step:
7232 case X86::BI__builtin_ia32_rdrand32_step:
7233 case X86::BI__builtin_ia32_rdrand64_step:
7234 case X86::BI__builtin_ia32_rdseed16_step:
7235 case X86::BI__builtin_ia32_rdseed32_step:
7236 case X86::BI__builtin_ia32_rdseed64_step: {
7238 switch (BuiltinID) {
7239 default: llvm_unreachable(
"Unsupported intrinsic!");
7240 case X86::BI__builtin_ia32_rdrand16_step:
7241 ID = Intrinsic::x86_rdrand_16;
7243 case X86::BI__builtin_ia32_rdrand32_step:
7244 ID = Intrinsic::x86_rdrand_32;
7246 case X86::BI__builtin_ia32_rdrand64_step:
7247 ID = Intrinsic::x86_rdrand_64;
7249 case X86::BI__builtin_ia32_rdseed16_step:
7250 ID = Intrinsic::x86_rdseed_16;
7252 case X86::BI__builtin_ia32_rdseed32_step:
7253 ID = Intrinsic::x86_rdseed_32;
7255 case X86::BI__builtin_ia32_rdseed64_step:
7256 ID = Intrinsic::x86_rdseed_64;
7263 return Builder.CreateExtractValue(Call, 1);
7267 case X86::BI__builtin_ia32_cmpeqps:
7268 case X86::BI__builtin_ia32_cmpeqpd:
7269 return getVectorFCmpIR(CmpInst::FCMP_OEQ);
7270 case X86::BI__builtin_ia32_cmpltps:
7271 case X86::BI__builtin_ia32_cmpltpd:
7272 return getVectorFCmpIR(CmpInst::FCMP_OLT);
7273 case X86::BI__builtin_ia32_cmpleps:
7274 case X86::BI__builtin_ia32_cmplepd:
7275 return getVectorFCmpIR(CmpInst::FCMP_OLE);
7276 case X86::BI__builtin_ia32_cmpunordps:
7277 case X86::BI__builtin_ia32_cmpunordpd:
7278 return getVectorFCmpIR(CmpInst::FCMP_UNO);
7279 case X86::BI__builtin_ia32_cmpneqps:
7280 case X86::BI__builtin_ia32_cmpneqpd:
7281 return getVectorFCmpIR(CmpInst::FCMP_UNE);
7282 case X86::BI__builtin_ia32_cmpnltps:
7283 case X86::BI__builtin_ia32_cmpnltpd:
7284 return getVectorFCmpIR(CmpInst::FCMP_UGE);
7285 case X86::BI__builtin_ia32_cmpnleps:
7286 case X86::BI__builtin_ia32_cmpnlepd:
7287 return getVectorFCmpIR(CmpInst::FCMP_UGT);
7288 case X86::BI__builtin_ia32_cmpordps:
7289 case X86::BI__builtin_ia32_cmpordpd:
7290 return getVectorFCmpIR(CmpInst::FCMP_ORD);
7291 case X86::BI__builtin_ia32_cmpps:
7292 case X86::BI__builtin_ia32_cmpps256:
7293 case X86::BI__builtin_ia32_cmppd:
7294 case X86::BI__builtin_ia32_cmppd256: {
7295 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7298 FCmpInst::Predicate Pred;
7300 case 0: Pred = FCmpInst::FCMP_OEQ;
break;
7301 case 1: Pred = FCmpInst::FCMP_OLT;
break;
7302 case 2: Pred = FCmpInst::FCMP_OLE;
break;
7303 case 3: Pred = FCmpInst::FCMP_UNO;
break;
7304 case 4: Pred = FCmpInst::FCMP_UNE;
break;
7305 case 5: Pred = FCmpInst::FCMP_UGE;
break;
7306 case 6: Pred = FCmpInst::FCMP_UGT;
break;
7307 case 7: Pred = FCmpInst::FCMP_ORD;
break;
7309 return getVectorFCmpIR(Pred);
7314 switch (BuiltinID) {
7315 default: llvm_unreachable(
"Unsupported intrinsic!");
7316 case X86::BI__builtin_ia32_cmpps:
7317 ID = Intrinsic::x86_sse_cmp_ps;
7319 case X86::BI__builtin_ia32_cmpps256:
7320 ID = Intrinsic::x86_avx_cmp_ps_256;
7322 case X86::BI__builtin_ia32_cmppd:
7323 ID = Intrinsic::x86_sse2_cmp_pd;
7325 case X86::BI__builtin_ia32_cmppd256:
7326 ID = Intrinsic::x86_avx_cmp_pd_256;
7334 case X86::BI__builtin_ia32_cmpeqss:
7335 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
7336 case X86::BI__builtin_ia32_cmpltss:
7337 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
7338 case X86::BI__builtin_ia32_cmpless:
7339 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
7340 case X86::BI__builtin_ia32_cmpunordss:
7341 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
7342 case X86::BI__builtin_ia32_cmpneqss:
7343 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
7344 case X86::BI__builtin_ia32_cmpnltss:
7345 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
7346 case X86::BI__builtin_ia32_cmpnless:
7347 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
7348 case X86::BI__builtin_ia32_cmpordss:
7349 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
7350 case X86::BI__builtin_ia32_cmpeqsd:
7351 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
7352 case X86::BI__builtin_ia32_cmpltsd:
7353 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
7354 case X86::BI__builtin_ia32_cmplesd:
7355 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
7356 case X86::BI__builtin_ia32_cmpunordsd:
7357 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
7358 case X86::BI__builtin_ia32_cmpneqsd:
7359 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
7360 case X86::BI__builtin_ia32_cmpnltsd:
7361 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
7362 case X86::BI__builtin_ia32_cmpnlesd:
7363 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
7364 case X86::BI__builtin_ia32_cmpordsd:
7365 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
7374 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
7379 switch (BuiltinID) {
7380 default:
return nullptr;
7384 case PPC::BI__builtin_ppc_get_timebase:
7388 case PPC::BI__builtin_altivec_lvx:
7389 case PPC::BI__builtin_altivec_lvxl:
7390 case PPC::BI__builtin_altivec_lvebx:
7391 case PPC::BI__builtin_altivec_lvehx:
7392 case PPC::BI__builtin_altivec_lvewx:
7393 case PPC::BI__builtin_altivec_lvsl:
7394 case PPC::BI__builtin_altivec_lvsr:
7395 case PPC::BI__builtin_vsx_lxvd2x:
7396 case PPC::BI__builtin_vsx_lxvw4x:
7400 Ops[0] =
Builder.CreateGEP(Ops[1], Ops[0]);
7403 switch (BuiltinID) {
7404 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
7405 case PPC::BI__builtin_altivec_lvx:
7406 ID = Intrinsic::ppc_altivec_lvx;
7408 case PPC::BI__builtin_altivec_lvxl:
7409 ID = Intrinsic::ppc_altivec_lvxl;
7411 case PPC::BI__builtin_altivec_lvebx:
7412 ID = Intrinsic::ppc_altivec_lvebx;
7414 case PPC::BI__builtin_altivec_lvehx:
7415 ID = Intrinsic::ppc_altivec_lvehx;
7417 case PPC::BI__builtin_altivec_lvewx:
7418 ID = Intrinsic::ppc_altivec_lvewx;
7420 case PPC::BI__builtin_altivec_lvsl:
7421 ID = Intrinsic::ppc_altivec_lvsl;
7423 case PPC::BI__builtin_altivec_lvsr:
7424 ID = Intrinsic::ppc_altivec_lvsr;
7426 case PPC::BI__builtin_vsx_lxvd2x:
7427 ID = Intrinsic::ppc_vsx_lxvd2x;
7429 case PPC::BI__builtin_vsx_lxvw4x:
7430 ID = Intrinsic::ppc_vsx_lxvw4x;
7434 return Builder.CreateCall(F, Ops,
"");
7438 case PPC::BI__builtin_altivec_stvx:
7439 case PPC::BI__builtin_altivec_stvxl:
7440 case PPC::BI__builtin_altivec_stvebx:
7441 case PPC::BI__builtin_altivec_stvehx:
7442 case PPC::BI__builtin_altivec_stvewx:
7443 case PPC::BI__builtin_vsx_stxvd2x:
7444 case PPC::BI__builtin_vsx_stxvw4x:
7447 Ops[1] =
Builder.CreateGEP(Ops[2], Ops[1]);
7450 switch (BuiltinID) {
7451 default: llvm_unreachable(
"Unsupported st intrinsic!");
7452 case PPC::BI__builtin_altivec_stvx:
7453 ID = Intrinsic::ppc_altivec_stvx;
7455 case PPC::BI__builtin_altivec_stvxl:
7456 ID = Intrinsic::ppc_altivec_stvxl;
7458 case PPC::BI__builtin_altivec_stvebx:
7459 ID = Intrinsic::ppc_altivec_stvebx;
7461 case PPC::BI__builtin_altivec_stvehx:
7462 ID = Intrinsic::ppc_altivec_stvehx;
7464 case PPC::BI__builtin_altivec_stvewx:
7465 ID = Intrinsic::ppc_altivec_stvewx;
7467 case PPC::BI__builtin_vsx_stxvd2x:
7468 ID = Intrinsic::ppc_vsx_stxvd2x;
7470 case PPC::BI__builtin_vsx_stxvw4x:
7471 ID = Intrinsic::ppc_vsx_stxvw4x;
7475 return Builder.CreateCall(F, Ops,
"");
7478 case PPC::BI__builtin_vsx_xvsqrtsp:
7479 case PPC::BI__builtin_vsx_xvsqrtdp: {
7484 return Builder.CreateCall(F, X);
7487 case PPC::BI__builtin_altivec_vclzb:
7488 case PPC::BI__builtin_altivec_vclzh:
7489 case PPC::BI__builtin_altivec_vclzw:
7490 case PPC::BI__builtin_altivec_vclzd: {
7493 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
7495 return Builder.CreateCall(F, {
X, Undef});
7498 case PPC::BI__builtin_vsx_xvcpsgnsp:
7499 case PPC::BI__builtin_vsx_xvcpsgndp: {
7505 return Builder.CreateCall(F, {
X, Y});
7508 case PPC::BI__builtin_vsx_xvrspip:
7509 case PPC::BI__builtin_vsx_xvrdpip:
7510 case PPC::BI__builtin_vsx_xvrdpim:
7511 case PPC::BI__builtin_vsx_xvrspim:
7512 case PPC::BI__builtin_vsx_xvrdpi:
7513 case PPC::BI__builtin_vsx_xvrspi:
7514 case PPC::BI__builtin_vsx_xvrdpic:
7515 case PPC::BI__builtin_vsx_xvrspic:
7516 case PPC::BI__builtin_vsx_xvrdpiz:
7517 case PPC::BI__builtin_vsx_xvrspiz: {
7520 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
7521 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
7523 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
7524 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
7526 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
7527 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
7529 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
7530 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
7532 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
7533 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
7536 return Builder.CreateCall(F, X);
7540 case PPC::BI__builtin_vsx_xvabsdp:
7541 case PPC::BI__builtin_vsx_xvabssp: {
7545 return Builder.CreateCall(F, X);
7549 case PPC::BI__builtin_vsx_xvmaddadp:
7550 case PPC::BI__builtin_vsx_xvmaddasp:
7551 case PPC::BI__builtin_vsx_xvnmaddadp:
7552 case PPC::BI__builtin_vsx_xvnmaddasp:
7553 case PPC::BI__builtin_vsx_xvmsubadp:
7554 case PPC::BI__builtin_vsx_xvmsubasp:
7555 case PPC::BI__builtin_vsx_xvnmsubadp:
7556 case PPC::BI__builtin_vsx_xvnmsubasp: {
7561 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
7563 switch (BuiltinID) {
7564 case PPC::BI__builtin_vsx_xvmaddadp:
7565 case PPC::BI__builtin_vsx_xvmaddasp:
7566 return Builder.CreateCall(F, {
X, Y, Z});
7567 case PPC::BI__builtin_vsx_xvnmaddadp:
7568 case PPC::BI__builtin_vsx_xvnmaddasp:
7569 return Builder.CreateFSub(Zero,
7570 Builder.CreateCall(F, {X, Y, Z}),
"sub");
7571 case PPC::BI__builtin_vsx_xvmsubadp:
7572 case PPC::BI__builtin_vsx_xvmsubasp:
7574 {
X, Y,
Builder.CreateFSub(Zero, Z,
"sub")});
7575 case PPC::BI__builtin_vsx_xvnmsubadp:
7576 case PPC::BI__builtin_vsx_xvnmsubasp:
7579 return Builder.CreateFSub(Zero, FsubRes,
"sub");
7581 llvm_unreachable(
"Unknown FMA operation");
7589 switch (BuiltinID) {
7590 case AMDGPU::BI__builtin_amdgcn_div_scale:
7591 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
7610 = FlagOutPtr.
getPointer()->getType()->getPointerElementType();
7616 case AMDGPU::BI__builtin_amdgcn_div_fmas:
7617 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
7626 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
7628 case AMDGPU::BI__builtin_amdgcn_div_fixup:
7629 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
7631 case AMDGPU::BI__builtin_amdgcn_trig_preop:
7632 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
7634 case AMDGPU::BI__builtin_amdgcn_rcp:
7635 case AMDGPU::BI__builtin_amdgcn_rcpf:
7637 case AMDGPU::BI__builtin_amdgcn_rsq:
7638 case AMDGPU::BI__builtin_amdgcn_rsqf:
7640 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
7641 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
7643 case AMDGPU::BI__builtin_amdgcn_sinf:
7645 case AMDGPU::BI__builtin_amdgcn_cosf:
7647 case AMDGPU::BI__builtin_amdgcn_log_clampf:
7649 case AMDGPU::BI__builtin_amdgcn_ldexp:
7650 case AMDGPU::BI__builtin_amdgcn_ldexpf:
7652 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
7653 case AMDGPU::BI__builtin_amdgcn_frexp_mantf: {
7656 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
7657 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
7660 case AMDGPU::BI__builtin_amdgcn_fract:
7661 case AMDGPU::BI__builtin_amdgcn_fractf:
7663 case AMDGPU::BI__builtin_amdgcn_lerp:
7665 case AMDGPU::BI__builtin_amdgcn_class:
7666 case AMDGPU::BI__builtin_amdgcn_classf:
7669 case AMDGPU::BI__builtin_amdgcn_read_exec: {
7670 CallInst *CI = cast<CallInst>(
7672 CI->setConvergent();
7677 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
7679 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
7681 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
7685 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
7686 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
7688 case AMDGPU::BI__builtin_r600_read_tidig_x:
7690 case AMDGPU::BI__builtin_r600_read_tidig_y:
7692 case AMDGPU::BI__builtin_r600_read_tidig_z:
7703 unsigned IntrinsicID,
7714 return CGF.
Builder.CreateExtractValue(Call, 0);
7719 switch (BuiltinID) {
7720 case SystemZ::BI__builtin_tbegin: {
7722 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
7724 return Builder.CreateCall(F, {TDB, Control});
7726 case SystemZ::BI__builtin_tbegin_nofloat: {
7728 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
7730 return Builder.CreateCall(F, {TDB, Control});
7732 case SystemZ::BI__builtin_tbeginc: {
7734 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
7736 return Builder.CreateCall(F, {TDB, Control});
7738 case SystemZ::BI__builtin_tabort: {
7743 case SystemZ::BI__builtin_non_tx_store: {
7747 return Builder.CreateCall(F, {Data, Address});
7755 case SystemZ::BI__builtin_s390_vpopctb:
7756 case SystemZ::BI__builtin_s390_vpopcth:
7757 case SystemZ::BI__builtin_s390_vpopctf:
7758 case SystemZ::BI__builtin_s390_vpopctg: {
7762 return Builder.CreateCall(F, X);
7765 case SystemZ::BI__builtin_s390_vclzb:
7766 case SystemZ::BI__builtin_s390_vclzh:
7767 case SystemZ::BI__builtin_s390_vclzf:
7768 case SystemZ::BI__builtin_s390_vclzg: {
7771 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
7773 return Builder.CreateCall(F, {
X, Undef});
7776 case SystemZ::BI__builtin_s390_vctzb:
7777 case SystemZ::BI__builtin_s390_vctzh:
7778 case SystemZ::BI__builtin_s390_vctzf:
7779 case SystemZ::BI__builtin_s390_vctzg: {
7782 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
7784 return Builder.CreateCall(F, {
X, Undef});
7787 case SystemZ::BI__builtin_s390_vfsqdb: {
7791 return Builder.CreateCall(F, X);
7793 case SystemZ::BI__builtin_s390_vfmadb: {
7799 return Builder.CreateCall(F, {
X, Y, Z});
7801 case SystemZ::BI__builtin_s390_vfmsdb: {
7806 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
7808 return Builder.CreateCall(F, {
X, Y,
Builder.CreateFSub(Zero, Z,
"sub")});
7810 case SystemZ::BI__builtin_s390_vflpdb: {
7814 return Builder.CreateCall(F, X);
7816 case SystemZ::BI__builtin_s390_vflndb: {
7819 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
7821 return Builder.CreateFSub(Zero,
Builder.CreateCall(F, X),
"sub");
7823 case SystemZ::BI__builtin_s390_vfidb: {
7827 llvm::APSInt M4, M5;
7830 assert(IsConstM4 && IsConstM5 &&
"Constant arg isn't actually constant?");
7831 (void)IsConstM4; (void)IsConstM5;
7835 switch (M4.getZExtValue()) {
7838 switch (M5.getZExtValue()) {
7844 switch (M5.getZExtValue()) {
7854 if (ID != Intrinsic::not_intrinsic) {
7856 return Builder.CreateCall(F, X);
7861 return Builder.CreateCall(F, {
X, M4Value, M5Value});
7866 #define INTRINSIC_WITH_CC(NAME) \
7867 case SystemZ::BI__builtin_##NAME: \
7868 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
7935 #undef INTRINSIC_WITH_CC
7944 auto MakeLdg = [&](
unsigned IntrinsicID) {
7950 CGM.
getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
7952 {Ptr, ConstantInt::get(
Builder.getInt32Ty(), Align.getQuantity())});
7955 switch (BuiltinID) {
7956 case NVPTX::BI__nvvm_atom_add_gen_i:
7957 case NVPTX::BI__nvvm_atom_add_gen_l:
7958 case NVPTX::BI__nvvm_atom_add_gen_ll:
7961 case NVPTX::BI__nvvm_atom_sub_gen_i:
7962 case NVPTX::BI__nvvm_atom_sub_gen_l:
7963 case NVPTX::BI__nvvm_atom_sub_gen_ll:
7966 case NVPTX::BI__nvvm_atom_and_gen_i:
7967 case NVPTX::BI__nvvm_atom_and_gen_l:
7968 case NVPTX::BI__nvvm_atom_and_gen_ll:
7971 case NVPTX::BI__nvvm_atom_or_gen_i:
7972 case NVPTX::BI__nvvm_atom_or_gen_l:
7973 case NVPTX::BI__nvvm_atom_or_gen_ll:
7976 case NVPTX::BI__nvvm_atom_xor_gen_i:
7977 case NVPTX::BI__nvvm_atom_xor_gen_l:
7978 case NVPTX::BI__nvvm_atom_xor_gen_ll:
7981 case NVPTX::BI__nvvm_atom_xchg_gen_i:
7982 case NVPTX::BI__nvvm_atom_xchg_gen_l:
7983 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
7986 case NVPTX::BI__nvvm_atom_max_gen_i:
7987 case NVPTX::BI__nvvm_atom_max_gen_l:
7988 case NVPTX::BI__nvvm_atom_max_gen_ll:
7991 case NVPTX::BI__nvvm_atom_max_gen_ui:
7992 case NVPTX::BI__nvvm_atom_max_gen_ul:
7993 case NVPTX::BI__nvvm_atom_max_gen_ull:
7996 case NVPTX::BI__nvvm_atom_min_gen_i:
7997 case NVPTX::BI__nvvm_atom_min_gen_l:
7998 case NVPTX::BI__nvvm_atom_min_gen_ll:
8001 case NVPTX::BI__nvvm_atom_min_gen_ui:
8002 case NVPTX::BI__nvvm_atom_min_gen_ul:
8003 case NVPTX::BI__nvvm_atom_min_gen_ull:
8006 case NVPTX::BI__nvvm_atom_cas_gen_i:
8007 case NVPTX::BI__nvvm_atom_cas_gen_l:
8008 case NVPTX::BI__nvvm_atom_cas_gen_ll:
8013 case NVPTX::BI__nvvm_atom_add_gen_f: {
8019 CGM.
getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
8020 return Builder.CreateCall(FnALAF32, {Ptr, Val});
8023 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
8028 return Builder.CreateCall(FnALI32, {Ptr, Val});
8031 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
8036 return Builder.CreateCall(FnALD32, {Ptr, Val});
8039 case NVPTX::BI__nvvm_ldg_c:
8040 case NVPTX::BI__nvvm_ldg_c2:
8041 case NVPTX::BI__nvvm_ldg_c4:
8042 case NVPTX::BI__nvvm_ldg_s:
8043 case NVPTX::BI__nvvm_ldg_s2:
8044 case NVPTX::BI__nvvm_ldg_s4:
8045 case NVPTX::BI__nvvm_ldg_i:
8046 case NVPTX::BI__nvvm_ldg_i2:
8047 case NVPTX::BI__nvvm_ldg_i4:
8048 case NVPTX::BI__nvvm_ldg_l:
8049 case NVPTX::BI__nvvm_ldg_ll:
8050 case NVPTX::BI__nvvm_ldg_ll2:
8051 case NVPTX::BI__nvvm_ldg_uc:
8052 case NVPTX::BI__nvvm_ldg_uc2:
8053 case NVPTX::BI__nvvm_ldg_uc4:
8054 case NVPTX::BI__nvvm_ldg_us:
8055 case NVPTX::BI__nvvm_ldg_us2:
8056 case NVPTX::BI__nvvm_ldg_us4:
8057 case NVPTX::BI__nvvm_ldg_ui:
8058 case NVPTX::BI__nvvm_ldg_ui2:
8059 case NVPTX::BI__nvvm_ldg_ui4:
8060 case NVPTX::BI__nvvm_ldg_ul:
8061 case NVPTX::BI__nvvm_ldg_ull:
8062 case NVPTX::BI__nvvm_ldg_ull2:
8066 return MakeLdg(Intrinsic::nvvm_ldg_global_i);
8067 case NVPTX::BI__nvvm_ldg_f:
8068 case NVPTX::BI__nvvm_ldg_f2:
8069 case NVPTX::BI__nvvm_ldg_f4:
8070 case NVPTX::BI__nvvm_ldg_d:
8071 case NVPTX::BI__nvvm_ldg_d2:
8072 return MakeLdg(Intrinsic::nvvm_ldg_global_f);
8080 switch (BuiltinID) {
8081 case WebAssembly::BI__builtin_wasm_current_memory: {
8084 return Builder.CreateCall(Callee);
8086 case WebAssembly::BI__builtin_wasm_grow_memory: {
8089 return Builder.CreateCall(Callee, X);
llvm::PointerType * Int8PtrPtrTy
unsigned getAddressSpace() const
Return the address space of this type.
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Defines the clang::ASTContext interface.
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
PointerType - C99 6.7.5.1 - Pointer Declarators.
A (possibly-)qualified type.
#define fma(__x, __y, __z)
CodeGenTypes & getTypes()
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
llvm::Module & getModule() const
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
AlignmentSource
The source of the alignment of an l-value; an expression of confidence in the alignment actually matc...
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LLVMContext & getLLVMContext()
const TargetInfo & getTarget() const
static const Builtin::Info BuiltinInfo[]
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
llvm::LoadInst * CreateDefaultAlignedLoad(llvm::Value *Addr, const llvm::Twine &Name="")
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, SmallVectorImpl< Value * > &Ops)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Type * FloatTy
float, double
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size...
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool IsRead, StringRef SysReg="")
The base class of the type hierarchy.
bool isBooleanType() const
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
bool isBlockPointerType() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, const CallExpr *E, llvm::Value *calleeValue)
llvm::Type * getElementType() const
Return the type of the values stored in this address.
const Expr * getCallee() const
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
ParmVarDecl - Represents a parameter to a function.
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
The collection of all-type qualifiers we support.
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1)
void __ovld prefetch(const __global char *p, size_t num_elements)
Prefetch num_elements * sizeof(gentype) bytes into the global cache.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
static llvm::VectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool V1Ty=false)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
llvm::IntegerType * Int64Ty
llvm::IntegerType * SizeTy
static bool NEONSIMDIntrinsicsProvenSorted
RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, CGCalleeInfo CalleeInfo=CGCalleeInfo(), llvm::Instruction **callOrInvoke=nullptr)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Instrinsic::ID and the expression node, where the return value is the result of the operation.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
Utility to insert an atomic instruction based on Instrinsic::ID and the expression node...
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[]
APValue Val
Val - This is the value the expression can be folded to.
#define INTRINSIC_WITH_CC(NAME)
const TemplateArgument & getArg(unsigned Idx) const
Retrieve a specific template argument as a type.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
llvm::PointerType * VoidPtrTy
static bool AArch64SISDIntrinsicsProvenSorted
Expr * IgnoreParenCasts() LLVM_READONLY
IgnoreParenCasts - Ignore parentheses and casts.
Scope - A scope is a transient data structure that is used while parsing the program.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
RValue EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
detail::InMemoryDirectory::const_iterator I
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Represents a prototype with parameter type info, e.g.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::Value *callee, const Twine &name="")
#define NEONMAP0(NameBase)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm.va_end.
llvm::Value * getPointer() const
#define copysign(__x, __y)
Expr - This represents one expression.
const char * getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
#define INTRINSIC_X86_XSAVE_ID(NAME)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
ASTContext & getContext() const
CharUnits getNaturalPointeeTypeAlignment(QualType T, AlignmentSource *Source=nullptr)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
void add(RValue rvalue, QualType type, bool needscopy=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
char __ovld __cnfn min(char x, char y)
Returns y if y < x, otherwise it returns x.
static SVal getValue(SVal val, SValBuilder &svalBuilder)
llvm::LLVMContext & getLLVMContext()
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, SmallVectorImpl< Value * > &Ops, unsigned Align)
llvm::IntegerType * Int32Ty
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
Address EmitPointerWithAlignment(const Expr *Addr, AlignmentSource *Source=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
static const NeonIntrinsicInfo ARMSIMDIntrinsicMap[]
bool EvaluateAsInt(llvm::APSInt &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p, To) is correct.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=None)
class LLVM_ALIGNAS(8) TemplateSpecializationType unsigned NumArgs
Represents a type template specialization; the template must be a class template, a type alias templa...
static const NeonIntrinsicInfo * findNeonIntrinsicInMap(ArrayRef< NeonIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
The result type of a method or function.
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
GlobalDecl - represents a global declaration.
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, SmallVectorImpl< Value * > &Ops, unsigned Align)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
Enumerates target-specific builtins in their own namespaces within namespace clang.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeSet ExtraAttrs=llvm::AttributeSet())
Create a new runtime function with the specified type and name.
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
ASTContext & getContext() const
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
bool hasSideEffects() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
llvm::IntegerType * Int16Ty
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
bool isIntegerConstantExpr(llvm::APSInt &Result, const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return true if this expression is a valid integer constant expression...
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
EltType getEltType() const
const T * castAs() const
Member-template castAs<specific type>.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
char __ovld __cnfn rotate(char v, char i)
For each element in v, the bits are shifted left by the number of bits given by the corresponding ele...
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression, because a __builtin_ms_va_list is a pointer to a char.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to .fabs().
llvm::Value * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
ast_type_traits::DynTypedNode Node
CGFunctionInfo - Class to encapsulate the information about a function definition.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return 0.
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LoadInst * CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
const TargetInfo * getAuxTargetInfo() const
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
detail::InMemoryDirectory::const_iterator E
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Flags to identify the types for overloaded Neon builtins.
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx)
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const NeonIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops)
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e...
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::PointerType * Int8PtrTy
void setNontemporal(bool Value)
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
BoundNodesTreeBuilder *const Builder
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Type * ConvertType(QualType T)
Builtin::Context & BuiltinInfo
Defines the clang::TargetInfo interface.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static RValue get(llvm::Value *V)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
LValue - This represents an lvalue references.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
CallArgList - Type for representing both the value and type of arguments in a call.
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
static bool AArch64SIMDIntrinsicsProvenSorted
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
bool isPointerType() const