80class AArch64FastISel final :
public FastISel {
83 using BaseKind =
enum {
89 BaseKind
Kind = RegBase;
95 unsigned OffsetReg = 0;
103 void setKind(BaseKind K) {
Kind =
K; }
104 BaseKind getKind()
const {
return Kind; }
107 bool isRegBase()
const {
return Kind == RegBase; }
108 bool isFIBase()
const {
return Kind == FrameIndexBase; }
110 void setReg(
unsigned Reg) {
111 assert(isRegBase() &&
"Invalid base register access!");
116 assert(isRegBase() &&
"Invalid base register access!");
120 void setOffsetReg(
unsigned Reg) {
124 unsigned getOffsetReg()
const {
128 void setFI(
unsigned FI) {
129 assert(isFIBase() &&
"Invalid base frame index access!");
133 unsigned getFI()
const {
134 assert(isFIBase() &&
"Invalid base frame index access!");
138 void setOffset(int64_t O) {
Offset =
O; }
140 void setShift(
unsigned S) { Shift = S; }
141 unsigned getShift() {
return Shift; }
170 bool selectRem(
const Instruction *
I,
unsigned ISDOpcode);
183 bool isTypeLegal(
Type *Ty,
MVT &VT);
184 bool isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed =
false);
185 bool isValueAvailable(
const Value *V)
const;
186 bool computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty =
nullptr);
187 bool computeCallAddress(
const Value *V, Address &
Addr);
188 bool simplifyAddress(Address &
Addr,
MVT VT);
193 bool tryEmitSmallMemCpy(Address Dest, Address Src,
uint64_t Len,
202 unsigned emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
203 const Value *RHS,
bool SetFlags =
false,
204 bool WantResult =
true,
bool IsZExt =
false);
205 unsigned emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
206 unsigned RHSReg,
bool SetFlags =
false,
207 bool WantResult =
true);
208 unsigned emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
209 uint64_t Imm,
bool SetFlags =
false,
210 bool WantResult =
true);
211 unsigned emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
213 uint64_t ShiftImm,
bool SetFlags =
false,
214 bool WantResult =
true);
215 unsigned emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
217 uint64_t ShiftImm,
bool SetFlags =
false,
218 bool WantResult =
true);
221 bool emitCompareAndBranch(
const BranchInst *BI);
223 bool emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
bool IsZExt);
224 bool emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
225 bool emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS);
230 bool emitStoreRelease(
MVT VT,
unsigned SrcReg,
unsigned AddrReg,
232 unsigned emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
bool isZExt);
233 unsigned emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool isZExt);
234 unsigned emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
235 bool SetFlags =
false,
bool WantResult =
true,
236 bool IsZExt =
false);
237 unsigned emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm);
238 unsigned emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
239 bool SetFlags =
false,
bool WantResult =
true,
240 bool IsZExt =
false);
241 unsigned emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
242 bool WantResult =
true);
243 unsigned emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
245 bool WantResult =
true);
246 unsigned emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
const Value *LHS,
248 unsigned emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
250 unsigned emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
251 unsigned RHSReg,
uint64_t ShiftImm);
252 unsigned emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
253 unsigned emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
254 unsigned emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
255 unsigned emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
256 unsigned emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
257 unsigned emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
259 unsigned emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
260 unsigned emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
262 unsigned emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
263 unsigned emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
264 bool IsZExt =
false);
275 bool finishCall(CallLoweringInfo &CLI,
MVT RetVT,
unsigned NumBytes);
292#include "AArch64GenFastISel.inc"
299 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
300 "Unexpected integer extend instruction.");
301 assert(!
I->getType()->isVectorTy() &&
I->getType()->isIntegerTy() &&
302 "Unexpected value type.");
303 bool IsZExt = isa<ZExtInst>(
I);
305 if (
const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0)))
309 if (
const auto *
Arg = dyn_cast<Argument>(
I->getOperand(0)))
310 if ((IsZExt &&
Arg->hasZExtAttr()) || (!IsZExt &&
Arg->hasSExtAttr()))
346unsigned AArch64FastISel::fastMaterializeAlloca(
const AllocaInst *AI) {
348 "Alloca should always return a pointer.");
351 if (!FuncInfo.StaticAllocaMap.count(AI))
355 FuncInfo.StaticAllocaMap.find(AI);
357 if (SI != FuncInfo.StaticAllocaMap.end()) {
358 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
359 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
370unsigned AArch64FastISel::materializeInt(
const ConstantInt *CI,
MVT VT) {
379 : &AArch64::GPR32RegClass;
380 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
381 Register ResultReg = createResultReg(RC);
382 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(TargetOpcode::COPY),
387unsigned AArch64FastISel::materializeFP(
const ConstantFP *CFP,
MVT VT) {
391 return fastMaterializeFloatZero(CFP);
403 unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
404 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
409 unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm;
411 &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
413 Register TmpReg = createResultReg(RC);
414 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc1), TmpReg)
417 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
418 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
419 TII.get(TargetOpcode::COPY), ResultReg)
429 unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Alignment);
430 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
431 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
434 unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui;
435 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
436 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
442unsigned AArch64FastISel::materializeGV(
const GlobalValue *GV) {
449 if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
452 unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
454 EVT DestEVT = TLI.getValueType(
DL, GV->
getType(),
true);
458 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
463 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
468 if (Subtarget->isTargetILP32()) {
469 ResultReg = createResultReg(&AArch64::GPR32RegClass);
470 LdrOpc = AArch64::LDRWui;
472 ResultReg = createResultReg(&AArch64::GPR64RegClass);
473 LdrOpc = AArch64::LDRXui;
475 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(LdrOpc),
480 if (!Subtarget->isTargetILP32())
485 Register Result64 = createResultReg(&AArch64::GPR64RegClass);
486 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
487 TII.get(TargetOpcode::SUBREG_TO_REG))
495 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
513 unsigned DstReg = createResultReg(&AArch64::GPR64commonRegClass);
514 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::MOVKXi),
523 ResultReg = createResultReg(&AArch64::GPR64spRegClass);
524 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
534unsigned AArch64FastISel::fastMaterializeConstant(
const Constant *
C) {
535 EVT CEVT = TLI.getValueType(
DL,
C->getType(),
true);
543 if (isa<ConstantPointerNull>(
C)) {
548 if (
const auto *CI = dyn_cast<ConstantInt>(
C))
549 return materializeInt(CI, VT);
550 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
551 return materializeFP(CFP, VT);
552 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(
C))
553 return materializeGV(GV);
558unsigned AArch64FastISel::fastMaterializeFloatZero(
const ConstantFP* CFP) {
560 "Floating-point constant is not a positive zero.");
562 if (!isTypeLegal(CFP->
getType(), VT))
569 unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
570 unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr;
571 return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg);
576 if (
const auto *
MI = dyn_cast<MulOperator>(
I)) {
577 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(0)))
578 if (
C->getValue().isPowerOf2())
580 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(1)))
581 if (
C->getValue().isPowerOf2())
588bool AArch64FastISel::computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty)
590 const User *
U =
nullptr;
591 unsigned Opcode = Instruction::UserOp1;
592 if (
const Instruction *
I = dyn_cast<Instruction>(Obj)) {
595 if (FuncInfo.StaticAllocaMap.count(
static_cast<const AllocaInst *
>(Obj)) ||
596 FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
597 Opcode =
I->getOpcode();
600 }
else if (
const ConstantExpr *
C = dyn_cast<ConstantExpr>(Obj)) {
601 Opcode =
C->getOpcode();
605 if (
auto *Ty = dyn_cast<PointerType>(Obj->
getType()))
606 if (Ty->getAddressSpace() > 255)
614 case Instruction::BitCast:
616 return computeAddress(
U->getOperand(0),
Addr, Ty);
618 case Instruction::IntToPtr:
620 if (TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
621 TLI.getPointerTy(
DL))
622 return computeAddress(
U->getOperand(0),
Addr, Ty);
625 case Instruction::PtrToInt:
627 if (TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
628 return computeAddress(
U->getOperand(0),
Addr, Ty);
631 case Instruction::GetElementPtr: {
639 const Value *
Op = GTI.getOperand();
640 if (
StructType *STy = GTI.getStructTypeOrNull()) {
642 unsigned Idx = cast<ConstantInt>(Op)->getZExtValue();
645 uint64_t S =
DL.getTypeAllocSize(GTI.getIndexedType());
647 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
652 if (canFoldAddIntoGEP(U, Op)) {
655 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
658 Op = cast<AddOperator>(Op)->getOperand(0);
662 goto unsupported_gep;
668 Addr.setOffset(TmpOffset);
669 if (computeAddress(
U->getOperand(0),
Addr, Ty))
678 case Instruction::Alloca: {
681 FuncInfo.StaticAllocaMap.find(AI);
682 if (SI != FuncInfo.StaticAllocaMap.end()) {
683 Addr.setKind(Address::FrameIndexBase);
689 case Instruction::Add: {
694 if (isa<ConstantInt>(LHS))
697 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
699 return computeAddress(LHS,
Addr, Ty);
703 if (computeAddress(LHS,
Addr, Ty) && computeAddress(RHS,
Addr, Ty))
709 case Instruction::Sub: {
714 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
716 return computeAddress(LHS,
Addr, Ty);
720 case Instruction::Shl: {
721 if (
Addr.getOffsetReg())
724 const auto *CI = dyn_cast<ConstantInt>(
U->getOperand(1));
729 if (Val < 1 || Val > 3)
735 NumBytes = NumBits / 8;
740 if (NumBytes != (1ULL << Val))
746 const Value *Src =
U->getOperand(0);
747 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
748 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
750 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
752 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
754 Src = ZE->getOperand(0);
756 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
758 SE->getOperand(0)->getType()->isIntegerTy(32)) {
760 Src = SE->getOperand(0);
766 if (
const auto *AI = dyn_cast<BinaryOperator>(Src))
767 if (AI->
getOpcode() == Instruction::And) {
771 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
772 if (
C->getValue() == 0xffffffff)
775 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
776 if (
C->getValue() == 0xffffffff) {
781 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, AArch64::sub_32);
782 Addr.setOffsetReg(Reg);
790 Addr.setOffsetReg(Reg);
793 case Instruction::Mul: {
794 if (
Addr.getOffsetReg())
804 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
805 if (
C->getValue().isPowerOf2())
808 assert(isa<ConstantInt>(RHS) &&
"Expected an ConstantInt.");
809 const auto *
C = cast<ConstantInt>(RHS);
810 unsigned Val =
C->getValue().logBase2();
811 if (Val < 1 || Val > 3)
817 NumBytes = NumBits / 8;
822 if (NumBytes != (1ULL << Val))
829 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
830 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
832 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
834 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
836 Src = ZE->getOperand(0);
838 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
840 SE->getOperand(0)->getType()->isIntegerTy(32)) {
842 Src = SE->getOperand(0);
851 Addr.setOffsetReg(Reg);
854 case Instruction::And: {
855 if (
Addr.getOffsetReg())
858 if (!Ty ||
DL.getTypeSizeInBits(Ty) != 8)
864 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
865 if (
C->getValue() == 0xffffffff)
868 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
869 if (
C->getValue() == 0xffffffff) {
877 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, AArch64::sub_32);
878 Addr.setOffsetReg(Reg);
883 case Instruction::SExt:
884 case Instruction::ZExt: {
885 if (!
Addr.getReg() ||
Addr.getOffsetReg())
888 const Value *Src =
nullptr;
890 if (
const auto *ZE = dyn_cast<ZExtInst>(U)) {
891 if (!
isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
893 Src = ZE->getOperand(0);
895 }
else if (
const auto *SE = dyn_cast<SExtInst>(U)) {
896 if (!
isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
898 Src = SE->getOperand(0);
909 Addr.setOffsetReg(Reg);
914 if (
Addr.isRegBase() && !
Addr.getReg()) {
922 if (!
Addr.getOffsetReg()) {
926 Addr.setOffsetReg(Reg);
933bool AArch64FastISel::computeCallAddress(
const Value *V, Address &
Addr) {
934 const User *
U =
nullptr;
935 unsigned Opcode = Instruction::UserOp1;
938 if (
const auto *
I = dyn_cast<Instruction>(V)) {
939 Opcode =
I->getOpcode();
941 InMBB =
I->getParent() == FuncInfo.MBB->getBasicBlock();
942 }
else if (
const auto *
C = dyn_cast<ConstantExpr>(V)) {
943 Opcode =
C->getOpcode();
949 case Instruction::BitCast:
952 return computeCallAddress(
U->getOperand(0),
Addr);
954 case Instruction::IntToPtr:
957 TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
958 TLI.getPointerTy(
DL))
959 return computeCallAddress(
U->getOperand(0),
Addr);
961 case Instruction::PtrToInt:
963 if (InMBB && TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
964 return computeCallAddress(
U->getOperand(0),
Addr);
968 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
969 Addr.setGlobalValue(GV);
974 if (!
Addr.getGlobalValue()) {
975 Addr.setReg(getRegForValue(V));
976 return Addr.getReg() != 0;
982bool AArch64FastISel::isTypeLegal(
Type *Ty,
MVT &VT) {
983 EVT evt = TLI.getValueType(
DL, Ty,
true);
985 if (Subtarget->isTargetILP32() && Ty->
isPointerTy())
999 return TLI.isTypeLegal(VT);
1006bool AArch64FastISel::isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed) {
1010 if (isTypeLegal(Ty, VT))
1021bool AArch64FastISel::isValueAvailable(
const Value *V)
const {
1022 if (!isa<Instruction>(V))
1025 const auto *
I = cast<Instruction>(V);
1026 return FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB;
1029bool AArch64FastISel::simplifyAddress(Address &
Addr,
MVT VT) {
1030 if (Subtarget->isTargetILP32())
1037 bool ImmediateOffsetNeedsLowering =
false;
1038 bool RegisterOffsetNeedsLowering =
false;
1041 ImmediateOffsetNeedsLowering =
true;
1042 else if (
Offset > 0 && !(
Offset & (ScaleFactor - 1)) &&
1043 !isUInt<12>(
Offset / ScaleFactor))
1044 ImmediateOffsetNeedsLowering =
true;
1049 if (!ImmediateOffsetNeedsLowering &&
Addr.getOffset() &&
Addr.getOffsetReg())
1050 RegisterOffsetNeedsLowering =
true;
1053 if (
Addr.isRegBase() &&
Addr.getOffsetReg() && !
Addr.getReg())
1054 RegisterOffsetNeedsLowering =
true;
1059 if ((ImmediateOffsetNeedsLowering ||
Addr.getOffsetReg()) &&
Addr.isFIBase())
1061 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
1062 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
1067 Addr.setKind(Address::RegBase);
1068 Addr.setReg(ResultReg);
1071 if (RegisterOffsetNeedsLowering) {
1072 unsigned ResultReg = 0;
1073 if (
Addr.getReg()) {
1076 ResultReg = emitAddSub_rx(
true,
MVT::i64,
Addr.getReg(),
1077 Addr.getOffsetReg(),
Addr.getExtendType(),
1080 ResultReg = emitAddSub_rs(
true,
MVT::i64,
Addr.getReg(),
1086 Addr.getShift(),
true);
1089 Addr.getShift(),
false);
1097 Addr.setReg(ResultReg);
1098 Addr.setOffsetReg(0);
1105 if (ImmediateOffsetNeedsLowering) {
1115 Addr.setReg(ResultReg);
1121void AArch64FastISel::addLoadStoreOperands(Address &
Addr,
1124 unsigned ScaleFactor,
1126 int64_t
Offset =
Addr.getOffset() / ScaleFactor;
1128 if (
Addr.isFIBase()) {
1129 int FI =
Addr.getFI();
1132 MMO = FuncInfo.MF->getMachineMemOperand(
1134 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
1138 assert(
Addr.isRegBase() &&
"Unexpected address kind.");
1145 if (
Addr.getOffsetReg()) {
1146 assert(
Addr.getOffset() == 0 &&
"Unexpected offset");
1161unsigned AArch64FastISel::emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
1162 const Value *RHS,
bool SetFlags,
1163 bool WantResult,
bool IsZExt) {
1165 bool NeedExtend =
false;
1188 if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
1192 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1197 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1198 if (
const auto *SI = dyn_cast<BinaryOperator>(LHS))
1199 if (isa<ConstantInt>(
SI->getOperand(1)))
1200 if (
SI->getOpcode() == Instruction::Shl ||
1201 SI->getOpcode() == Instruction::LShr ||
1202 SI->getOpcode() == Instruction::AShr )
1205 Register LHSReg = getRegForValue(LHS);
1210 LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
1212 unsigned ResultReg = 0;
1213 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1214 uint64_t Imm = IsZExt ?
C->getZExtValue() :
C->getSExtValue();
1215 if (
C->isNegative())
1216 ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,
1219 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,
1221 }
else if (
const auto *
C = dyn_cast<Constant>(RHS))
1222 if (
C->isNullValue())
1223 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);
1230 isValueAvailable(RHS)) {
1231 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS))
1232 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1)))
1233 if ((
SI->getOpcode() == Instruction::Shl) && (
C->getZExtValue() < 4)) {
1234 Register RHSReg = getRegForValue(
SI->getOperand(0));
1237 return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType,
1238 C->getZExtValue(), SetFlags, WantResult);
1240 Register RHSReg = getRegForValue(RHS);
1243 return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,
1244 SetFlags, WantResult);
1250 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1251 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1253 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1254 if (
C->getValue().isPowerOf2())
1257 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1258 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1259 Register RHSReg = getRegForValue(MulLHS);
1262 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg,
AArch64_AM::LSL,
1263 ShiftVal, SetFlags, WantResult);
1271 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
1272 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1274 switch (
SI->getOpcode()) {
1282 Register RHSReg = getRegForValue(
SI->getOperand(0));
1285 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,
1286 ShiftVal, SetFlags, WantResult);
1294 Register RHSReg = getRegForValue(RHS);
1299 RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
1301 return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);
1304unsigned AArch64FastISel::emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1305 unsigned RHSReg,
bool SetFlags,
1307 assert(LHSReg && RHSReg &&
"Invalid register number.");
1309 if (LHSReg == AArch64::SP || LHSReg == AArch64::WSP ||
1310 RHSReg == AArch64::SP || RHSReg == AArch64::WSP)
1316 static const unsigned OpcTable[2][2][2] = {
1317 { { AArch64::SUBWrr, AArch64::SUBXrr },
1318 { AArch64::ADDWrr, AArch64::ADDXrr } },
1319 { { AArch64::SUBSWrr, AArch64::SUBSXrr },
1320 { AArch64::ADDSWrr, AArch64::ADDSXrr } }
1323 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1325 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1328 ResultReg = createResultReg(RC);
1330 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1335 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1341unsigned AArch64FastISel::emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1344 assert(LHSReg &&
"Invalid register number.");
1350 if (isUInt<12>(Imm))
1352 else if ((Imm & 0xfff000) == Imm) {
1358 static const unsigned OpcTable[2][2][2] = {
1359 { { AArch64::SUBWri, AArch64::SUBXri },
1360 { AArch64::ADDWri, AArch64::ADDXri } },
1361 { { AArch64::SUBSWri, AArch64::SUBSXri },
1362 { AArch64::ADDSWri, AArch64::ADDSXri } }
1365 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1368 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1370 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1373 ResultReg = createResultReg(RC);
1375 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1379 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1386unsigned AArch64FastISel::emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1391 assert(LHSReg && RHSReg &&
"Invalid register number.");
1392 assert(LHSReg != AArch64::SP && LHSReg != AArch64::WSP &&
1393 RHSReg != AArch64::SP && RHSReg != AArch64::WSP);
1402 static const unsigned OpcTable[2][2][2] = {
1403 { { AArch64::SUBWrs, AArch64::SUBXrs },
1404 { AArch64::ADDWrs, AArch64::ADDXrs } },
1405 { { AArch64::SUBSWrs, AArch64::SUBSXrs },
1406 { AArch64::ADDSWrs, AArch64::ADDSXrs } }
1409 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1411 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1414 ResultReg = createResultReg(RC);
1416 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1421 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1424 .
addImm(getShifterImm(ShiftType, ShiftImm));
1428unsigned AArch64FastISel::emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1433 assert(LHSReg && RHSReg &&
"Invalid register number.");
1434 assert(LHSReg != AArch64::XZR && LHSReg != AArch64::WZR &&
1435 RHSReg != AArch64::XZR && RHSReg != AArch64::WZR);
1443 static const unsigned OpcTable[2][2][2] = {
1444 { { AArch64::SUBWrx, AArch64::SUBXrx },
1445 { AArch64::ADDWrx, AArch64::ADDXrx } },
1446 { { AArch64::SUBSWrx, AArch64::SUBSXrx },
1447 { AArch64::ADDSWrx, AArch64::ADDSXrx } }
1450 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1453 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1455 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1458 ResultReg = createResultReg(RC);
1460 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1465 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1468 .
addImm(getArithExtendImm(ExtType, ShiftImm));
1472bool AArch64FastISel::emitCmp(
const Value *LHS,
const Value *RHS,
bool IsZExt) {
1474 EVT EVT = TLI.getValueType(
DL, Ty,
true);
1487 return emitICmp(VT, LHS, RHS, IsZExt);
1490 return emitFCmp(VT, LHS, RHS);
1494bool AArch64FastISel::emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1496 return emitSub(RetVT, LHS, RHS,
true,
false,
1500bool AArch64FastISel::emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm) {
1501 return emitAddSub_ri(
false, RetVT, LHSReg, Imm,
1505bool AArch64FastISel::emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS) {
1511 bool UseImm =
false;
1512 if (
const auto *CFP = dyn_cast<ConstantFP>(RHS))
1516 Register LHSReg = getRegForValue(LHS);
1521 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;
1522 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1527 Register RHSReg = getRegForValue(RHS);
1531 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;
1532 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1538unsigned AArch64FastISel::emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1539 bool SetFlags,
bool WantResult,
bool IsZExt) {
1540 return emitAddSub(
true, RetVT, LHS, RHS, SetFlags, WantResult,
1549unsigned AArch64FastISel::emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm) {
1552 ResultReg = emitAddSub_ri(
false, VT, Op0, -Imm);
1554 ResultReg = emitAddSub_ri(
true, VT, Op0, Imm);
1563 ResultReg = emitAddSub_rr(
true, VT, Op0, CReg);
1567unsigned AArch64FastISel::emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1568 bool SetFlags,
bool WantResult,
bool IsZExt) {
1569 return emitAddSub(
false, RetVT, LHS, RHS, SetFlags, WantResult,
1573unsigned AArch64FastISel::emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
1574 unsigned RHSReg,
bool WantResult) {
1575 return emitAddSub_rr(
false, RetVT, LHSReg, RHSReg,
1579unsigned AArch64FastISel::emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
1582 uint64_t ShiftImm,
bool WantResult) {
1583 return emitAddSub_rs(
false, RetVT, LHSReg, RHSReg, ShiftType,
1584 ShiftImm,
true, WantResult);
1587unsigned AArch64FastISel::emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
1590 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
1600 if (
const auto *SI = dyn_cast<ShlOperator>(LHS))
1601 if (isa<ConstantInt>(
SI->getOperand(1)))
1604 Register LHSReg = getRegForValue(LHS);
1608 unsigned ResultReg = 0;
1609 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1611 ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);
1619 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1620 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1622 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1623 if (
C->getValue().isPowerOf2())
1626 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1627 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1629 Register RHSReg = getRegForValue(MulLHS);
1632 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1640 if (
const auto *SI = dyn_cast<ShlOperator>(RHS))
1641 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1643 Register RHSReg = getRegForValue(
SI->getOperand(0));
1646 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1652 Register RHSReg = getRegForValue(RHS);
1657 ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, RHSReg);
1660 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
1665unsigned AArch64FastISel::emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
1668 "ISD nodes are not consecutive!");
1669 static const unsigned OpcTable[3][2] = {
1670 { AArch64::ANDWri, AArch64::ANDXri },
1671 { AArch64::ORRWri, AArch64::ORRXri },
1672 { AArch64::EORWri, AArch64::EORXri }
1685 Opc = OpcTable[
Idx][0];
1686 RC = &AArch64::GPR32spRegClass;
1691 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1692 RC = &AArch64::GPR64spRegClass;
1701 fastEmitInst_ri(Opc, RC, LHSReg,
1705 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
1710unsigned AArch64FastISel::emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
1711 unsigned LHSReg,
unsigned RHSReg,
1714 "ISD nodes are not consecutive!");
1715 static const unsigned OpcTable[3][2] = {
1716 { AArch64::ANDWrs, AArch64::ANDXrs },
1717 { AArch64::ORRWrs, AArch64::ORRXrs },
1718 { AArch64::EORWrs, AArch64::EORXrs }
1734 Opc = OpcTable[ISDOpc -
ISD::AND][0];
1735 RC = &AArch64::GPR32RegClass;
1738 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1739 RC = &AArch64::GPR64RegClass;
1743 fastEmitInst_rri(Opc, RC, LHSReg, RHSReg,
1747 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
1752unsigned AArch64FastISel::emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
1754 return emitLogicalOp_ri(
ISD::AND, RetVT, LHSReg, Imm);
1757unsigned AArch64FastISel::emitLoad(
MVT VT,
MVT RetVT, Address
Addr,
1759 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1763 if (!simplifyAddress(
Addr, VT))
1772 bool UseScaled =
true;
1773 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
1778 static const unsigned GPOpcTable[2][8][4] = {
1780 { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi,
1782 { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi,
1784 { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui,
1786 { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui,
1788 { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX,
1790 { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX,
1792 { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW,
1794 { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW,
1798 { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1800 { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1802 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1804 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1806 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1808 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1810 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1812 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1817 static const unsigned FPOpcTable[4][2] = {
1818 { AArch64::LDURSi, AArch64::LDURDi },
1819 { AArch64::LDRSui, AArch64::LDRDui },
1820 { AArch64::LDRSroX, AArch64::LDRDroX },
1821 { AArch64::LDRSroW, AArch64::LDRDroW }
1826 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
1827 Addr.getOffsetReg();
1828 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
1833 bool IsRet64Bit = RetVT ==
MVT::i64;
1839 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][0];
1840 RC = (IsRet64Bit && !WantZExt) ?
1841 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1844 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][1];
1845 RC = (IsRet64Bit && !WantZExt) ?
1846 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1849 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][2];
1850 RC = (IsRet64Bit && !WantZExt) ?
1851 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1854 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][3];
1855 RC = &AArch64::GPR64RegClass;
1858 Opc = FPOpcTable[
Idx][0];
1859 RC = &AArch64::FPR32RegClass;
1862 Opc = FPOpcTable[
Idx][1];
1863 RC = &AArch64::FPR64RegClass;
1868 Register ResultReg = createResultReg(RC);
1870 TII.get(Opc), ResultReg);
1875 unsigned ANDReg = emitAnd_ri(
MVT::i32, ResultReg, 1);
1876 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
1883 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
1884 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1885 TII.get(AArch64::SUBREG_TO_REG), Reg64)
1888 .
addImm(AArch64::sub_32);
1894bool AArch64FastISel::selectAddSub(
const Instruction *
I) {
1896 if (!isTypeSupported(
I->getType(), VT,
true))
1900 return selectOperator(
I,
I->getOpcode());
1903 switch (
I->getOpcode()) {
1906 case Instruction::Add:
1907 ResultReg = emitAdd(VT,
I->getOperand(0),
I->getOperand(1));
1909 case Instruction::Sub:
1910 ResultReg = emitSub(VT,
I->getOperand(0),
I->getOperand(1));
1916 updateValueMap(
I, ResultReg);
1920bool AArch64FastISel::selectLogicalOp(
const Instruction *
I) {
1922 if (!isTypeSupported(
I->getType(), VT,
true))
1926 return selectOperator(
I,
I->getOpcode());
1929 switch (
I->getOpcode()) {
1932 case Instruction::And:
1933 ResultReg = emitLogicalOp(
ISD::AND, VT,
I->getOperand(0),
I->getOperand(1));
1935 case Instruction::Or:
1936 ResultReg = emitLogicalOp(
ISD::OR, VT,
I->getOperand(0),
I->getOperand(1));
1938 case Instruction::Xor:
1939 ResultReg = emitLogicalOp(
ISD::XOR, VT,
I->getOperand(0),
I->getOperand(1));
1945 updateValueMap(
I, ResultReg);
1949bool AArch64FastISel::selectLoad(
const Instruction *
I) {
1954 if (!isTypeSupported(
I->getType(), VT,
true) ||
1955 cast<LoadInst>(
I)->isAtomic())
1958 const Value *SV =
I->getOperand(0);
1959 if (TLI.supportSwiftError()) {
1962 if (
const Argument *
Arg = dyn_cast<Argument>(SV)) {
1963 if (
Arg->hasSwiftErrorAttr())
1967 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
1968 if (Alloca->isSwiftError())
1975 if (!computeAddress(
I->getOperand(0),
Addr,
I->getType()))
1979 bool WantZExt =
true;
1981 const Value *IntExtVal =
nullptr;
1982 if (
I->hasOneUse()) {
1983 if (
const auto *ZE = dyn_cast<ZExtInst>(
I->use_begin()->getUser())) {
1984 if (isTypeSupported(ZE->getType(), RetVT))
1988 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I->use_begin()->getUser())) {
1989 if (isTypeSupported(SE->getType(), RetVT))
1997 unsigned ResultReg =
1998 emitLoad(VT, RetVT,
Addr, WantZExt, createMachineMemOperandFor(
I));
2019 auto *
MI =
MRI.getUniqueVRegDef(Reg);
2025 ResultReg = std::prev(
I)->getOperand(0).getReg();
2026 removeDeadCode(
I, std::next(
I));
2028 ResultReg = fastEmitInst_extractsubreg(
MVT::i32, ResultReg,
2031 updateValueMap(
I, ResultReg);
2040 for (
auto &Opnd :
MI->uses()) {
2042 Reg = Opnd.getReg();
2047 removeDeadCode(
I, std::next(
I));
2050 MI =
MRI.getUniqueVRegDef(Reg);
2052 updateValueMap(IntExtVal, ResultReg);
2056 updateValueMap(
I, ResultReg);
2060bool AArch64FastISel::emitStoreRelease(
MVT VT,
unsigned SrcReg,
2065 default:
return false;
2066 case MVT::i8: Opc = AArch64::STLRB;
break;
2067 case MVT::i16: Opc = AArch64::STLRH;
break;
2068 case MVT::i32: Opc = AArch64::STLRW;
break;
2069 case MVT::i64: Opc = AArch64::STLRX;
break;
2075 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2082bool AArch64FastISel::emitStore(
MVT VT,
unsigned SrcReg, Address
Addr,
2084 if (!TLI.allowsMisalignedMemoryAccesses(VT))
2088 if (!simplifyAddress(
Addr, VT))
2097 bool UseScaled =
true;
2098 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
2103 static const unsigned OpcTable[4][6] = {
2104 { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi,
2105 AArch64::STURSi, AArch64::STURDi },
2106 { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui,
2107 AArch64::STRSui, AArch64::STRDui },
2108 { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX,
2109 AArch64::STRSroX, AArch64::STRDroX },
2110 { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW,
2111 AArch64::STRSroW, AArch64::STRDroW }
2115 bool VTIsi1 =
false;
2116 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
2117 Addr.getOffsetReg();
2118 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
2125 case MVT::i1: VTIsi1 =
true; [[fallthrough]];
2135 if (VTIsi1 && SrcReg != AArch64::WZR) {
2136 unsigned ANDReg = emitAnd_ri(
MVT::i32, SrcReg, 1);
2137 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2144 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(SrcReg);
2150bool AArch64FastISel::selectStore(
const Instruction *
I) {
2152 const Value *Op0 =
I->getOperand(0);
2156 if (!isTypeSupported(Op0->
getType(), VT,
true))
2159 const Value *PtrV =
I->getOperand(1);
2160 if (TLI.supportSwiftError()) {
2163 if (
const Argument *
Arg = dyn_cast<Argument>(PtrV)) {
2164 if (
Arg->hasSwiftErrorAttr())
2168 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
2169 if (Alloca->isSwiftError())
2176 unsigned SrcReg = 0;
2177 if (
const auto *CI = dyn_cast<ConstantInt>(Op0)) {
2179 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2180 }
else if (
const auto *CF = dyn_cast<ConstantFP>(Op0)) {
2181 if (CF->isZero() && !CF->isNegative()) {
2183 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2188 SrcReg = getRegForValue(Op0);
2193 auto *
SI = cast<StoreInst>(
I);
2196 if (
SI->isAtomic()) {
2201 Register AddrReg = getRegForValue(PtrV);
2202 return emitStoreRelease(VT, SrcReg, AddrReg,
2203 createMachineMemOperandFor(
I));
2264bool AArch64FastISel::emitCompareAndBranch(
const BranchInst *BI) {
2268 if (FuncInfo.MF->getFunction().hasFnAttribute(
2269 Attribute::SpeculativeLoadHardening))
2291 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2298 switch (Predicate) {
2303 if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue())
2306 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2309 if (
const auto *AI = dyn_cast<BinaryOperator>(LHS))
2310 if (AI->
getOpcode() == Instruction::And && isValueAvailable(AI)) {
2314 if (
const auto *
C = dyn_cast<ConstantInt>(AndLHS))
2315 if (
C->getValue().isPowerOf2())
2318 if (
const auto *
C = dyn_cast<ConstantInt>(AndRHS))
2319 if (
C->getValue().isPowerOf2()) {
2320 TestBit =
C->getValue().logBase2();
2332 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2340 if (!isa<ConstantInt>(RHS))
2343 if (cast<ConstantInt>(RHS)->getValue() !=
APInt(BW, -1,
true))
2351 static const unsigned OpcTable[2][2][2] = {
2352 { {AArch64::CBZW, AArch64::CBZX },
2353 {AArch64::CBNZW, AArch64::CBNZX} },
2354 { {AArch64::TBZW, AArch64::TBZX },
2355 {AArch64::TBNZW, AArch64::TBNZX} }
2358 bool IsBitTest = TestBit != -1;
2359 bool Is64Bit = BW == 64;
2360 if (TestBit < 32 && TestBit >= 0)
2363 unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
2366 Register SrcReg = getRegForValue(LHS);
2370 if (BW == 64 && !Is64Bit)
2371 SrcReg = fastEmitInst_extractsubreg(
MVT::i32, SrcReg, AArch64::sub_32);
2373 if ((BW < 32) && !IsBitTest)
2374 SrcReg = emitIntExt(VT, SrcReg,
MVT::i32,
true);
2379 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
2389bool AArch64FastISel::selectBranch(
const Instruction *
I) {
2401 if (CI->
hasOneUse() && isValueAvailable(CI)) {
2404 switch (Predicate) {
2408 fastEmitBranch(FBB, MIMD.getDL());
2411 fastEmitBranch(
TBB, MIMD.getDL());
2416 if (emitCompareAndBranch(BI))
2420 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2433 switch (Predicate) {
2449 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2455 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2462 }
else if (
const auto *CI = dyn_cast<ConstantInt>(BI->
getCondition())) {
2465 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::B))
2474 FuncInfo.MBB->addSuccessorWithoutProb(
Target);
2486 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2500 unsigned Opcode = AArch64::TBNZW;
2501 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2503 Opcode = AArch64::TBZW;
2509 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2510 .
addReg(ConstrainedCondReg)
2518bool AArch64FastISel::selectIndirectBr(
const Instruction *
I) {
2527 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(AddrReg);
2531 FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[Succ]);
2537 const CmpInst *CI = cast<CmpInst>(
I);
2545 unsigned ResultReg = 0;
2546 switch (Predicate) {
2550 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2551 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2552 TII.get(TargetOpcode::COPY), ResultReg)
2561 updateValueMap(
I, ResultReg);
2569 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2573 static unsigned CondCodeTable[2][2] = {
2578 switch (Predicate) {
2590 Register TmpReg1 = createResultReg(&AArch64::GPR32RegClass);
2591 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2596 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2602 updateValueMap(
I, ResultReg);
2610 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2616 updateValueMap(
I, ResultReg);
2622bool AArch64FastISel::optimizeSelect(
const SelectInst *SI) {
2623 if (!
SI->getType()->isIntegerTy(1))
2626 const Value *Src1Val, *Src2Val;
2628 bool NeedExtraOp =
false;
2629 if (
auto *CI = dyn_cast<ConstantInt>(
SI->getTrueValue())) {
2631 Src1Val =
SI->getCondition();
2632 Src2Val =
SI->getFalseValue();
2633 Opc = AArch64::ORRWrr;
2636 Src1Val =
SI->getFalseValue();
2637 Src2Val =
SI->getCondition();
2638 Opc = AArch64::BICWrr;
2640 }
else if (
auto *CI = dyn_cast<ConstantInt>(
SI->getFalseValue())) {
2642 Src1Val =
SI->getCondition();
2643 Src2Val =
SI->getTrueValue();
2644 Opc = AArch64::ORRWrr;
2648 Src1Val =
SI->getCondition();
2649 Src2Val =
SI->getTrueValue();
2650 Opc = AArch64::ANDWrr;
2657 Register Src1Reg = getRegForValue(Src1Val);
2661 Register Src2Reg = getRegForValue(Src2Val);
2668 Register ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
2670 updateValueMap(SI, ResultReg);
2674bool AArch64FastISel::selectSelect(
const Instruction *
I) {
2675 assert(isa<SelectInst>(
I) &&
"Expected a select instruction.");
2677 if (!isTypeSupported(
I->getType(), VT))
2689 Opc = AArch64::CSELWr;
2690 RC = &AArch64::GPR32RegClass;
2693 Opc = AArch64::CSELXr;
2694 RC = &AArch64::GPR64RegClass;
2697 Opc = AArch64::FCSELSrrr;
2698 RC = &AArch64::FPR32RegClass;
2701 Opc = AArch64::FCSELDrrr;
2702 RC = &AArch64::FPR64RegClass;
2711 if (optimizeSelect(SI))
2715 if (foldXALUIntrinsic(
CC,
I,
Cond)) {
2721 isValueAvailable(
Cond)) {
2722 const auto *
Cmp = cast<CmpInst>(
Cond);
2725 const Value *FoldSelect =
nullptr;
2726 switch (Predicate) {
2730 FoldSelect =
SI->getFalseValue();
2733 FoldSelect =
SI->getTrueValue();
2738 Register SrcReg = getRegForValue(FoldSelect);
2742 updateValueMap(
I, SrcReg);
2752 switch (Predicate) {
2774 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II,
2780 Register Src1Reg = getRegForValue(
SI->getTrueValue());
2781 Register Src2Reg = getRegForValue(
SI->getFalseValue());
2783 if (!Src1Reg || !Src2Reg)
2787 Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg, ExtraCC);
2789 Register ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg,
CC);
2790 updateValueMap(
I, ResultReg);
2794bool AArch64FastISel::selectFPExt(
const Instruction *
I) {
2796 if (!
I->getType()->isDoubleTy() || !
V->getType()->isFloatTy())
2803 Register ResultReg = createResultReg(&AArch64::FPR64RegClass);
2804 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTDSr),
2806 updateValueMap(
I, ResultReg);
2810bool AArch64FastISel::selectFPTrunc(
const Instruction *
I) {
2812 if (!
I->getType()->isFloatTy() || !
V->getType()->isDoubleTy())
2819 Register ResultReg = createResultReg(&AArch64::FPR32RegClass);
2820 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTSDr),
2822 updateValueMap(
I, ResultReg);
2829 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2832 Register SrcReg = getRegForValue(
I->getOperand(0));
2836 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2843 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;
2845 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;
2848 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;
2850 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;
2852 Register ResultReg = createResultReg(
2853 DestVT ==
MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2854 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
2856 updateValueMap(
I, ResultReg);
2862 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2869 "Unexpected value type.");
2871 Register SrcReg = getRegForValue(
I->getOperand(0));
2875 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2888 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;
2890 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;
2893 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;
2895 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;
2898 Register ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg);
2899 updateValueMap(
I, ResultReg);
2903bool AArch64FastISel::fastLowerArguments() {
2904 if (!FuncInfo.CanLowerReturn)
2915 if (Subtarget->hasCustomCallingConv())
2919 unsigned GPRCnt = 0;
2920 unsigned FPRCnt = 0;
2921 for (
auto const &
Arg :
F->args()) {
2922 if (
Arg.hasAttribute(Attribute::ByVal) ||
2923 Arg.hasAttribute(Attribute::InReg) ||
2924 Arg.hasAttribute(Attribute::StructRet) ||
2925 Arg.hasAttribute(Attribute::SwiftSelf) ||
2926 Arg.hasAttribute(Attribute::SwiftAsync) ||
2927 Arg.hasAttribute(Attribute::SwiftError) ||
2928 Arg.hasAttribute(Attribute::Nest))
2935 EVT ArgVT = TLI.getValueType(
DL, ArgTy);
2944 (!Subtarget->hasNEON() || !Subtarget->isLittleEndian()))
2955 if (GPRCnt > 8 || FPRCnt > 8)
2960 { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4,
2961 AArch64::W5, AArch64::W6, AArch64::W7 },
2962 { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4,
2963 AArch64::X5, AArch64::X6, AArch64::X7 },
2964 { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4,
2965 AArch64::H5, AArch64::H6, AArch64::H7 },
2966 { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4,
2967 AArch64::S5, AArch64::S6, AArch64::S7 },
2968 { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4,
2969 AArch64::D5, AArch64::D6, AArch64::D7 },
2970 { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4,
2971 AArch64::Q5, AArch64::Q6, AArch64::Q7 }
2975 unsigned FPRIdx = 0;
2976 for (
auto const &
Arg :
F->args()) {
2977 MVT VT = TLI.getSimpleValueType(
DL,
Arg.getType());
2982 RC = &AArch64::GPR32RegClass;
2986 RC = &AArch64::GPR64RegClass;
2989 RC = &AArch64::FPR16RegClass;
2992 RC = &AArch64::FPR32RegClass;
2995 RC = &AArch64::FPR64RegClass;
2998 RC = &AArch64::FPR128RegClass;
3002 Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
3006 Register ResultReg = createResultReg(RC);
3007 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3008 TII.get(TargetOpcode::COPY), ResultReg)
3010 updateValueMap(&
Arg, ResultReg);
3015bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
3017 unsigned &NumBytes) {
3020 CCState CCInfo(
CC,
false, *FuncInfo.MF, ArgLocs, *Context);
3021 CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(
CC));
3024 NumBytes = CCInfo.getNextStackOffset();
3027 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
3028 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackDown))
3033 const Value *ArgVal = CLI.OutVals[VA.getValNo()];
3034 MVT ArgVT = OutVTs[VA.getValNo()];
3036 Register ArgReg = getRegForValue(ArgVal);
3041 switch (VA.getLocInfo()) {
3045 MVT DestVT = VA.getLocVT();
3047 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
false);
3055 MVT DestVT = VA.getLocVT();
3057 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
true);
3067 if (VA.isRegLoc() && !VA.needsCustom()) {
3068 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3069 TII.get(TargetOpcode::COPY), VA.getLocReg()).
addReg(ArgReg);
3070 CLI.OutRegs.push_back(VA.getLocReg());
3071 }
else if (VA.needsCustom()) {
3075 assert(VA.isMemLoc() &&
"Assuming store on stack.");
3078 if (isa<UndefValue>(ArgVal))
3084 unsigned BEAlign = 0;
3085 if (ArgSize < 8 && !Subtarget->isLittleEndian())
3086 BEAlign = 8 - ArgSize;
3089 Addr.setKind(Address::RegBase);
3090 Addr.setReg(AArch64::SP);
3091 Addr.setOffset(VA.getLocMemOffset() + BEAlign);
3105bool AArch64FastISel::finishCall(CallLoweringInfo &CLI,
MVT RetVT,
3106 unsigned NumBytes) {
3110 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3111 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackUp))
3117 CCState CCInfo(
CC,
false, *FuncInfo.MF, RVLocs, *Context);
3118 CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(
CC));
3121 if (RVLocs.
size() != 1)
3125 MVT CopyVT = RVLocs[0].getValVT();
3128 if (CopyVT.
isVector() && !Subtarget->isLittleEndian())
3131 Register ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
3132 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3133 TII.get(TargetOpcode::COPY), ResultReg)
3134 .
addReg(RVLocs[0].getLocReg());
3135 CLI.InRegs.push_back(RVLocs[0].getLocReg());
3137 CLI.ResultReg = ResultReg;
3138 CLI.NumResultRegs = 1;
3144bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3146 bool IsTailCall = CLI.IsTailCall;
3147 bool IsVarArg = CLI.IsVarArg;
3156 if (CLI.CB && CLI.CB->hasFnAttr(Attribute::ReturnsTwice) &&
3157 !Subtarget->noBTIAtReturnTwice() &&
3162 if (CLI.CB && CLI.CB->isIndirectCall() &&
3172 if (Subtarget->isTargetILP32())
3190 if (CLI.RetTy->isVoidTy())
3192 else if (!isTypeLegal(CLI.RetTy, RetVT))
3195 for (
auto Flag : CLI.OutFlags)
3197 Flag.isSwiftSelf() ||
Flag.isSwiftAsync() ||
Flag.isSwiftError())
3202 OutVTs.
reserve(CLI.OutVals.size());
3204 for (
auto *Val : CLI.OutVals) {
3206 if (!isTypeLegal(Val->getType(), VT) &&
3224 if (Subtarget->isTargetWindows() &&
Addr.getGlobalValue() &&
3225 Addr.getGlobalValue()->hasExternalWeakLinkage())
3230 if (!processCallArgs(CLI, OutVTs, NumBytes))
3234 if (
RegInfo->isAnyArgRegReserved(*MF))
3235 RegInfo->emitReservedArgRegCallError(*MF);
3239 if (Subtarget->useSmallAddressing()) {
3242 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II);
3245 else if (
Addr.getGlobalValue())
3247 else if (
Addr.getReg()) {
3253 unsigned CallReg = 0;
3255 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
3256 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
3260 CallReg = createResultReg(&AArch64::GPR64RegClass);
3261 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3262 TII.get(AArch64::LDRXui), CallReg)
3266 }
else if (
Addr.getGlobalValue())
3267 CallReg = materializeGV(
Addr.getGlobalValue());
3268 else if (
Addr.getReg())
3269 CallReg =
Addr.getReg();
3276 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(CallReg);
3280 for (
auto Reg : CLI.OutRegs)
3290 return finishCall(CLI, RetVT, NumBytes);
3295 return Len / Alignment->value() <= 4;
3300bool AArch64FastISel::tryEmitSmallMemCpy(Address Dest, Address Src,
3303 if (!isMemCpySmall(Len, Alignment))
3306 int64_t UnscaledOffset = 0;
3312 if (!Alignment || *Alignment >= 8) {
3323 assert(Alignment &&
"Alignment is set in this branch");
3325 if (Len >= 4 && *Alignment == 4)
3327 else if (Len >= 2 && *Alignment == 2)
3334 unsigned ResultReg =
emitLoad(VT, VT, Src);
3343 UnscaledOffset +=
Size;
3346 Dest.setOffset(OrigDest.getOffset() + UnscaledOffset);
3347 Src.setOffset(OrigSrc.getOffset() + UnscaledOffset);
3358 if (!isa<ExtractValueInst>(
Cond))
3361 const auto *EV = cast<ExtractValueInst>(
Cond);
3362 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
3365 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
3369 cast<StructType>(
Callee->getReturnType())->getTypeAtIndex(0U);
3370 if (!isTypeLegal(
RetTy, RetVT))
3376 const Value *
LHS = II->getArgOperand(0);
3377 const Value *
RHS = II->getArgOperand(1);
3380 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && II->isCommutative())
3388 case Intrinsic::smul_with_overflow:
3389 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3390 if (
C->getValue() == 2)
3391 IID = Intrinsic::sadd_with_overflow;
3393 case Intrinsic::umul_with_overflow:
3394 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3395 if (
C->getValue() == 2)
3396 IID = Intrinsic::uadd_with_overflow;
3404 case Intrinsic::sadd_with_overflow:
3405 case Intrinsic::ssub_with_overflow:
3408 case Intrinsic::uadd_with_overflow:
3411 case Intrinsic::usub_with_overflow:
3414 case Intrinsic::smul_with_overflow:
3415 case Intrinsic::umul_with_overflow:
3421 if (!isValueAvailable(II))
3427 for (
auto Itr = std::prev(Start); Itr != End; --Itr) {
3430 if (!isa<ExtractValueInst>(Itr))
3434 const auto *EVI = cast<ExtractValueInst>(Itr);
3435 if (EVI->getAggregateOperand() != II)
3443bool AArch64FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *II) {
3446 default:
return false;
3447 case Intrinsic::frameaddress: {
3453 Register SrcReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3454 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3462 unsigned Depth = cast<ConstantInt>(II->
getOperand(0))->getZExtValue();
3464 DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
3466 assert(DestReg &&
"Unexpected LDR instruction emission failure.");
3470 updateValueMap(II, SrcReg);
3473 case Intrinsic::sponentry: {
3478 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
3479 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3480 TII.get(AArch64::ADDXri), ResultReg)
3485 updateValueMap(II, ResultReg);
3488 case Intrinsic::memcpy:
3489 case Intrinsic::memmove: {
3490 const auto *MTI = cast<MemTransferInst>(II);
3492 if (MTI->isVolatile())
3498 if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) {
3501 uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue();
3503 if (MTI->getDestAlign() || MTI->getSourceAlign())
3504 Alignment = std::min(MTI->getDestAlign().valueOrOne(),
3505 MTI->getSourceAlign().valueOrOne());
3506 if (isMemCpySmall(Len, Alignment)) {
3508 if (!computeAddress(MTI->getRawDest(), Dest) ||
3509 !computeAddress(MTI->getRawSource(), Src))
3511 if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment))
3516 if (!MTI->getLength()->getType()->isIntegerTy(64))
3519 if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255)
3524 const char *IntrMemName = isa<MemCpyInst>(II) ?
"memcpy" :
"memmove";
3525 return lowerCallTo(II, IntrMemName, II->
arg_size() - 1);
3527 case Intrinsic::memset: {
3528 const MemSetInst *MSI = cast<MemSetInst>(II);
3541 return lowerCallTo(II,
"memset", II->
arg_size() - 1);
3543 case Intrinsic::sin:
3544 case Intrinsic::cos:
3545 case Intrinsic::pow: {
3547 if (!isTypeLegal(II->
getType(), RetVT))
3554 { RTLIB::SIN_F32, RTLIB::SIN_F64 },
3555 { RTLIB::COS_F32, RTLIB::COS_F64 },
3556 { RTLIB::POW_F32, RTLIB::POW_F64 }
3563 case Intrinsic::sin:
3564 LC = LibCallTable[0][Is64Bit];
3566 case Intrinsic::cos:
3567 LC = LibCallTable[1][Is64Bit];
3569 case Intrinsic::pow:
3570 LC = LibCallTable[2][Is64Bit];
3578 for (
auto &
Arg : II->
args()) {
3581 Entry.Ty =
Arg->getType();
3582 Args.push_back(Entry);
3585 CallLoweringInfo CLI;
3587 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC), II->
getType(),
3588 TLI.getLibcallName(LC), std::move(Args));
3589 if (!lowerCallTo(CLI))
3591 updateValueMap(II, CLI.ResultReg);
3594 case Intrinsic::fabs: {
3596 if (!isTypeLegal(II->
getType(), VT))
3604 Opc = AArch64::FABSSr;
3607 Opc = AArch64::FABSDr;
3613 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3614 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
3616 updateValueMap(II, ResultReg);
3619 case Intrinsic::trap:
3620 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3623 case Intrinsic::debugtrap:
3624 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3628 case Intrinsic::sqrt: {
3632 if (!isTypeLegal(
RetTy, VT))
3639 unsigned ResultReg = fastEmit_r(VT, VT,
ISD::FSQRT, Op0Reg);
3643 updateValueMap(II, ResultReg);
3646 case Intrinsic::sadd_with_overflow:
3647 case Intrinsic::uadd_with_overflow:
3648 case Intrinsic::ssub_with_overflow:
3649 case Intrinsic::usub_with_overflow:
3650 case Intrinsic::smul_with_overflow:
3651 case Intrinsic::umul_with_overflow: {
3654 auto *Ty = cast<StructType>(
Callee->getReturnType());
3658 if (!isTypeLegal(
RetTy, VT))
3667 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && II->
isCommutative())
3675 case Intrinsic::smul_with_overflow:
3676 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3677 if (
C->getValue() == 2) {
3678 IID = Intrinsic::sadd_with_overflow;
3682 case Intrinsic::umul_with_overflow:
3683 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3684 if (
C->getValue() == 2) {
3685 IID = Intrinsic::uadd_with_overflow;
3691 unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
3695 case Intrinsic::sadd_with_overflow:
3696 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3699 case Intrinsic::uadd_with_overflow:
3700 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3703 case Intrinsic::ssub_with_overflow:
3704 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3707 case Intrinsic::usub_with_overflow:
3708 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3711 case Intrinsic::smul_with_overflow: {
3713 Register LHSReg = getRegForValue(LHS);
3717 Register RHSReg = getRegForValue(RHS);
3722 MulReg = emitSMULL_rr(
MVT::i64, LHSReg, RHSReg);
3724 fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3726 emitAddSub_rx(
false,
MVT::i64, MulReg, MulSubReg,
3734 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3735 unsigned SMULHReg = fastEmit_rr(VT, VT,
ISD::MULHS, LHSReg, RHSReg);
3741 case Intrinsic::umul_with_overflow: {
3743 Register LHSReg = getRegForValue(LHS);
3747 Register RHSReg = getRegForValue(RHS);
3752 MulReg = emitUMULL_rr(
MVT::i64, LHSReg, RHSReg);
3754 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3755 TII.get(AArch64::ANDSXri), AArch64::XZR)
3758 MulReg = fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3763 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3764 unsigned UMULHReg = fastEmit_rr(VT, VT,
ISD::MULHU, LHSReg, RHSReg);
3765 emitSubs_rr(VT, AArch64::XZR, UMULHReg,
false);
3772 ResultReg1 = createResultReg(TLI.getRegClassFor(VT));
3773 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3774 TII.get(TargetOpcode::COPY), ResultReg1).
addReg(MulReg);
3780 ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
3781 AArch64::WZR, AArch64::WZR,
3782 getInvertedCondCode(
CC));
3784 assert((ResultReg1 + 1) == ResultReg2 &&
3785 "Nonconsecutive result registers.");
3786 updateValueMap(II, ResultReg1, 2);
3795 const Function &
F = *
I->getParent()->getParent();
3797 if (!FuncInfo.CanLowerReturn)
3803 if (TLI.supportSwiftError() &&
3804 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
3807 if (TLI.supportSplitCSR(FuncInfo.MF))
3813 if (
Ret->getNumOperands() > 0) {
3820 CCState CCInfo(
CC,
F.isVarArg(), *FuncInfo.MF, ValLocs,
I->getContext());
3823 CCInfo.AnalyzeReturn(Outs, RetCC);
3826 if (ValLocs.
size() != 1)
3830 const Value *RV =
Ret->getOperand(0);
3848 if (!
MRI.getRegClass(SrcReg)->contains(DestReg))
3857 !Subtarget->isLittleEndian())
3866 if (RVVT != DestVT) {
3870 if (!Outs[0].
Flags.isZExt() && !Outs[0].Flags.isSExt())
3873 bool IsZExt = Outs[0].Flags.isZExt();
3874 SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
3882 SrcReg = emitAnd_ri(
MVT::i64, SrcReg, 0xffffffff);
3885 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3886 TII.get(TargetOpcode::COPY), DestReg).
addReg(SrcReg);
3893 TII.get(AArch64::RET_ReallyLR));
3894 for (
unsigned RetReg : RetRegs)
3899bool AArch64FastISel::selectTrunc(
const Instruction *
I) {
3900 Type *DestTy =
I->getType();
3902 Type *SrcTy =
Op->getType();
3904 EVT SrcEVT = TLI.getValueType(
DL, SrcTy,
true);
3905 EVT DestEVT = TLI.getValueType(
DL, DestTy,
true);
3921 Register SrcReg = getRegForValue(Op);
3951 ResultReg = emitAnd_ri(
MVT::i32, Reg32, Mask);
3952 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3954 ResultReg = createResultReg(&AArch64::GPR32RegClass);
3955 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3956 TII.get(TargetOpcode::COPY), ResultReg)
3960 updateValueMap(
I, ResultReg);
3964unsigned AArch64FastISel::emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool IsZExt) {
3967 "Unexpected value type.");
3973 unsigned ResultReg = emitAnd_ri(
MVT::i32, SrcReg, 1);
3974 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3978 Register Reg64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3979 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3980 TII.get(AArch64::SUBREG_TO_REG), Reg64)
3983 .
addImm(AArch64::sub_32);
3992 return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
3997unsigned AArch64FastISel::emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4005 Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR;
break;
4007 Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR;
break;
4011 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4012 return fastEmitInst_rrr(Opc, RC, Op0, Op1, ZReg);
4015unsigned AArch64FastISel::emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4019 return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
4020 Op0, Op1, AArch64::XZR);
4023unsigned AArch64FastISel::emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4027 return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
4028 Op0, Op1, AArch64::XZR);
4031unsigned AArch64FastISel::emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
4034 bool NeedTrunc =
false;
4038 case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4039 case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4040 case MVT::i32: Opc = AArch64::LSLVWr;
break;
4041 case MVT::i64: Opc = AArch64::LSLVXr;
break;
4045 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4047 Op1Reg = emitAnd_ri(
MVT::i32, Op1Reg, Mask);
4049 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4051 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
4055unsigned AArch64FastISel::emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4058 "Unexpected source/return type pair.");
4061 "Unexpected source value type.");
4063 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4065 bool Is64Bit = (RetVT ==
MVT::i64);
4066 unsigned RegSize = Is64Bit ? 64 : 32;
4070 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4074 if (RetVT == SrcVT) {
4075 Register ResultReg = createResultReg(RC);
4076 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4077 TII.get(TargetOpcode::COPY), ResultReg)
4081 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4085 if (Shift >= DstBits)
4113 unsigned ImmR =
RegSize - Shift;
4115 unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift);
4116 static const unsigned OpcTable[2][2] = {
4117 {AArch64::SBFMWri, AArch64::SBFMXri},
4118 {AArch64::UBFMWri, AArch64::UBFMXri}
4120 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4123 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4124 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4127 .
addImm(AArch64::sub_32);
4130 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4133unsigned AArch64FastISel::emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
4136 bool NeedTrunc =
false;
4140 case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4141 case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4142 case MVT::i32: Opc = AArch64::LSRVWr;
break;
4143 case MVT::i64: Opc = AArch64::LSRVXr;
break;
4147 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4149 Op0Reg = emitAnd_ri(
MVT::i32, Op0Reg, Mask);
4150 Op1Reg = emitAnd_ri(
MVT::i32, Op1Reg, Mask);
4152 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4154 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
4158unsigned AArch64FastISel::emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4161 "Unexpected source/return type pair.");
4164 "Unexpected source value type.");
4166 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4168 bool Is64Bit = (RetVT ==
MVT::i64);
4169 unsigned RegSize = Is64Bit ? 64 : 32;
4173 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4177 if (RetVT == SrcVT) {
4178 Register ResultReg = createResultReg(RC);
4179 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4180 TII.get(TargetOpcode::COPY), ResultReg)
4184 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4188 if (Shift >= DstBits)
4216 if (Shift >= SrcBits && IsZExt)
4222 Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4230 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4231 unsigned ImmS = SrcBits - 1;
4232 static const unsigned OpcTable[2][2] = {
4233 {AArch64::SBFMWri, AArch64::SBFMXri},
4234 {AArch64::UBFMWri, AArch64::UBFMXri}
4236 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4239 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4240 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4243 .
addImm(AArch64::sub_32);
4246 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4249unsigned AArch64FastISel::emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
4252 bool NeedTrunc =
false;
4256 case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4257 case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4258 case MVT::i32: Opc = AArch64::ASRVWr;
break;
4259 case MVT::i64: Opc = AArch64::ASRVXr;
break;
4263 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4265 Op0Reg = emitIntExt(RetVT, Op0Reg,
MVT::i32,
false);
4266 Op1Reg = emitAnd_ri(
MVT::i32, Op1Reg, Mask);
4268 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4270 ResultReg = emitAnd_ri(
MVT::i32, ResultReg, Mask);
4274unsigned AArch64FastISel::emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4277 "Unexpected source/return type pair.");
4280 "Unexpected source value type.");
4282 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4284 bool Is64Bit = (RetVT ==
MVT::i64);
4285 unsigned RegSize = Is64Bit ? 64 : 32;
4289 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4293 if (RetVT == SrcVT) {
4294 Register ResultReg = createResultReg(RC);
4295 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4296 TII.get(TargetOpcode::COPY), ResultReg)
4300 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4304 if (Shift >= DstBits)
4332 if (Shift >= SrcBits && IsZExt)
4335 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4336 unsigned ImmS = SrcBits - 1;
4337 static const unsigned OpcTable[2][2] = {
4338 {AArch64::SBFMWri, AArch64::SBFMXri},
4339 {AArch64::UBFMWri, AArch64::UBFMXri}
4341 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4344 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4345 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4348 .
addImm(AArch64::sub_32);
4351 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4354unsigned AArch64FastISel::emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
4375 return emiti1Ext(SrcReg, DestVT, IsZExt);
4378 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4380 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4385 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4387 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4392 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4401 Register Src64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4402 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4403 TII.get(AArch64::SUBREG_TO_REG), Src64)
4406 .
addImm(AArch64::sub_32);
4411 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4412 return fastEmitInst_rii(Opc, RC, SrcReg, 0, Imm);
4419 case AArch64::LDURBBi:
4420 case AArch64::LDURHHi:
4421 case AArch64::LDURWi:
4422 case AArch64::LDRBBui:
4423 case AArch64::LDRHHui:
4424 case AArch64::LDRWui:
4425 case AArch64::LDRBBroX:
4426 case AArch64::LDRHHroX:
4427 case AArch64::LDRWroX:
4428 case AArch64::LDRBBroW:
4429 case AArch64::LDRHHroW:
4430 case AArch64::LDRWroW:
4439 case AArch64::LDURSBWi:
4440 case AArch64::LDURSHWi:
4441 case AArch64::LDURSBXi:
4442 case AArch64::LDURSHXi:
4443 case AArch64::LDURSWi:
4444 case AArch64::LDRSBWui:
4445 case AArch64::LDRSHWui:
4446 case AArch64::LDRSBXui:
4447 case AArch64::LDRSHXui:
4448 case AArch64::LDRSWui:
4449 case AArch64::LDRSBWroX:
4450 case AArch64::LDRSHWroX:
4451 case AArch64::LDRSBXroX:
4452 case AArch64::LDRSHXroX:
4453 case AArch64::LDRSWroX:
4454 case AArch64::LDRSBWroW:
4455 case AArch64::LDRSHWroW:
4456 case AArch64::LDRSBXroW:
4457 case AArch64::LDRSHXroW:
4458 case AArch64::LDRSWroW:
4463bool AArch64FastISel::optimizeIntExtLoad(
const Instruction *
I,
MVT RetVT,
4465 const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0));
4466 if (!LI || !LI->hasOneUse())
4480 bool IsZExt = isa<ZExtInst>(
I);
4481 const auto *LoadMI =
MI;
4482 if (LoadMI->getOpcode() == TargetOpcode::COPY &&
4483 LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) {
4484 Register LoadReg =
MI->getOperand(1).getReg();
4485 LoadMI =
MRI.getUniqueVRegDef(LoadReg);
4486 assert(LoadMI &&
"Expected valid instruction");
4493 updateValueMap(
I, Reg);
4498 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
4499 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4500 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4503 .
addImm(AArch64::sub_32);
4506 assert((
MI->getOpcode() == TargetOpcode::COPY &&
4507 MI->getOperand(1).getSubReg() == AArch64::sub_32) &&
4508 "Expected copy instruction");
4509 Reg =
MI->getOperand(1).getReg();
4511 removeDeadCode(
I, std::next(
I));
4513 updateValueMap(
I, Reg);
4517bool AArch64FastISel::selectIntExt(
const Instruction *
I) {
4518 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
4519 "Unexpected integer extend instruction.");
4522 if (!isTypeSupported(
I->getType(), RetVT))
4525 if (!isTypeSupported(
I->getOperand(0)->getType(), SrcVT))
4529 if (optimizeIntExtLoad(
I, RetVT, SrcVT))
4532 Register SrcReg = getRegForValue(
I->getOperand(0));
4537 bool IsZExt = isa<ZExtInst>(
I);
4538 if (
const auto *
Arg = dyn_cast<Argument>(
I->getOperand(0))) {
4539 if ((IsZExt &&
Arg->hasZExtAttr()) || (!IsZExt &&
Arg->hasSExtAttr())) {
4541 Register ResultReg = createResultReg(&AArch64::GPR64RegClass);
4542 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4543 TII.get(AArch64::SUBREG_TO_REG), ResultReg)
4546 .
addImm(AArch64::sub_32);
4550 updateValueMap(
I, SrcReg);
4555 unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
4559 updateValueMap(
I, ResultReg);
4563bool AArch64FastISel::selectRem(
const Instruction *
I,
unsigned ISDOpcode) {
4564 EVT DestEVT = TLI.getValueType(
DL,
I->getType(),
true);
4573 bool Is64bit = (DestVT ==
MVT::i64);
4574 switch (ISDOpcode) {
4578 DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr;
4581 DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr;
4584 unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr;
4585 Register Src0Reg = getRegForValue(
I->getOperand(0));
4589 Register Src1Reg = getRegForValue(
I->getOperand(1));
4594 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4595 Register QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg, Src1Reg);
4596 assert(QuotReg &&
"Unexpected DIV instruction emission failure.");
4599 Register ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg, Src1Reg, Src0Reg);
4600 updateValueMap(
I, ResultReg);
4606 if (!isTypeSupported(
I->getType(), VT,
true))
4612 const Value *Src0 =
I->getOperand(0);
4613 const Value *Src1 =
I->getOperand(1);
4614 if (
const auto *
C = dyn_cast<ConstantInt>(Src0))
4615 if (
C->getValue().isPowerOf2())
4619 if (
const auto *
C = dyn_cast<ConstantInt>(Src1))
4620 if (
C->getValue().isPowerOf2()) {
4621 uint64_t ShiftVal =
C->getValue().logBase2();
4624 if (
const auto *ZExt = dyn_cast<ZExtInst>(Src0)) {
4627 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) {
4630 Src0 = ZExt->getOperand(0);
4633 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Src0)) {
4636 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) {
4639 Src0 = SExt->getOperand(0);
4644 Register Src0Reg = getRegForValue(Src0);
4648 unsigned ResultReg =
4649 emitLSL_ri(VT, SrcVT, Src0Reg, ShiftVal, IsZExt);
4652 updateValueMap(
I, ResultReg);
4657 Register Src0Reg = getRegForValue(
I->getOperand(0));
4661 Register Src1Reg = getRegForValue(
I->getOperand(1));
4665 unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src1Reg);
4670 updateValueMap(
I, ResultReg);
4674bool AArch64FastISel::selectShift(
const Instruction *
I) {
4676 if (!isTypeSupported(
I->getType(), RetVT,
true))
4680 return selectOperator(
I,
I->getOpcode());
4682 if (
const auto *
C = dyn_cast<ConstantInt>(
I->getOperand(1))) {
4683 unsigned ResultReg = 0;
4686 bool IsZExt =
I->getOpcode() != Instruction::AShr;
4687 const Value *Op0 =
I->getOperand(0);
4688 if (
const auto *ZExt = dyn_cast<ZExtInst>(Op0)) {
4691 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) {
4694 Op0 = ZExt->getOperand(0);
4697 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Op0)) {
4700 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) {
4703 Op0 = SExt->getOperand(0);
4708 Register Op0Reg = getRegForValue(Op0);
4712 switch (
I->getOpcode()) {
4714 case Instruction::Shl:
4715 ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4717 case Instruction::AShr:
4718 ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4720 case Instruction::LShr:
4721 ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4727 updateValueMap(
I, ResultReg);
4731 Register Op0Reg = getRegForValue(
I->getOperand(0));
4735 Register Op1Reg = getRegForValue(
I->getOperand(1));
4739 unsigned ResultReg = 0;
4740 switch (
I->getOpcode()) {
4742 case Instruction::Shl:
4743 ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);
4745 case Instruction::AShr:
4746 ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);
4748 case Instruction::LShr:
4749 ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);
4756 updateValueMap(
I, ResultReg);
4760bool AArch64FastISel::selectBitCast(
const Instruction *
I) {
4763 if (!isTypeLegal(
I->getOperand(0)->getType(), SrcVT))
4765 if (!isTypeLegal(
I->getType(), RetVT))
4770 Opc = AArch64::FMOVWSr;
4772 Opc = AArch64::FMOVXDr;
4774 Opc = AArch64::FMOVSWr;
4776 Opc = AArch64::FMOVDXr;
4783 case MVT::i32: RC = &AArch64::GPR32RegClass;
break;
4784 case MVT::i64: RC = &AArch64::GPR64RegClass;
break;
4785 case MVT::f32: RC = &AArch64::FPR32RegClass;
break;
4786 case MVT::f64: RC = &AArch64::FPR64RegClass;
break;
4788 Register Op0Reg = getRegForValue(
I->getOperand(0));
4792 Register ResultReg = fastEmitInst_r(Opc, RC, Op0Reg);
4796 updateValueMap(
I, ResultReg);
4800bool AArch64FastISel::selectFRem(
const Instruction *
I) {
4802 if (!isTypeLegal(
I->getType(), RetVT))
4810 LC = RTLIB::REM_F32;
4813 LC = RTLIB::REM_F64;
4818 Args.reserve(
I->getNumOperands());
4821 for (
auto &
Arg :
I->operands()) {
4824 Entry.Ty =
Arg->getType();
4825 Args.push_back(Entry);
4828 CallLoweringInfo CLI;
4830 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC),
I->getType(),
4831 TLI.getLibcallName(LC), std::move(Args));
4832 if (!lowerCallTo(CLI))
4834 updateValueMap(
I, CLI.ResultReg);
4838bool AArch64FastISel::selectSDiv(
const Instruction *
I) {
4840 if (!isTypeLegal(
I->getType(), VT))
4843 if (!isa<ConstantInt>(
I->getOperand(1)))
4846 const APInt &
C = cast<ConstantInt>(
I->getOperand(1))->getValue();
4848 !(
C.isPowerOf2() ||
C.isNegatedPowerOf2()))
4851 unsigned Lg2 =
C.countr_zero();
4852 Register Src0Reg = getRegForValue(
I->getOperand(0));
4856 if (cast<BinaryOperator>(
I)->isExact()) {
4857 unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Lg2);
4860 updateValueMap(
I, ResultReg);
4864 int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
4865 unsigned AddReg = emitAdd_ri_(VT, Src0Reg, Pow2MinusOne);
4870 if (!emitICmp_ri(VT, Src0Reg, 0))
4876 SelectOpc = AArch64::CSELXr;
4877 RC = &AArch64::GPR64RegClass;
4879 SelectOpc = AArch64::CSELWr;
4880 RC = &AArch64::GPR32RegClass;
4882 Register SelectReg = fastEmitInst_rri(SelectOpc, RC, AddReg, Src0Reg,
4889 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
4892 ResultReg = emitAddSub_rs(
false, VT, ZeroReg, SelectReg,
4895 ResultReg = emitASR_ri(VT, VT, SelectReg, Lg2);
4900 updateValueMap(
I, ResultReg);
4907unsigned AArch64FastISel::getRegForGEPIndex(
const Value *
Idx) {
4914 MVT PtrVT = TLI.getPointerTy(
DL);
4916 if (IdxVT.
bitsLT(PtrVT)) {
4917 IdxN = emitIntExt(IdxVT.
getSimpleVT(), IdxN, PtrVT,
false);
4918 }
else if (IdxVT.
bitsGT(PtrVT))
4919 llvm_unreachable(
"AArch64 FastISel doesn't support types larger than i64");
4927bool AArch64FastISel::selectGetElementPtr(
const Instruction *
I) {
4928 if (Subtarget->isTargetILP32())
4931 Register N = getRegForValue(
I->getOperand(0));
4938 MVT VT = TLI.getPointerTy(
DL);
4941 const Value *
Idx = GTI.getOperand();
4942 if (
auto *StTy = GTI.getStructTypeOrNull()) {
4943 unsigned Field = cast<ConstantInt>(
Idx)->getZExtValue();
4946 TotalOffs +=
DL.getStructLayout(StTy)->getElementOffset(
Field);
4948 Type *Ty = GTI.getIndexedType();
4951 if (
const auto *CI = dyn_cast<ConstantInt>(
Idx)) {
4956 DL.getTypeAllocSize(Ty) * cast<ConstantInt>(CI)->getSExtValue();
4960 N = emitAdd_ri_(VT,
N, TotalOffs);
4967 uint64_t ElementSize =
DL.getTypeAllocSize(Ty);
4968 unsigned IdxN = getRegForGEPIndex(
Idx);
4972 if (ElementSize != 1) {
4976 IdxN = emitMul_rr(VT, IdxN,
C);
4986 N = emitAdd_ri_(VT,
N, TotalOffs);
4990 updateValueMap(
I,
N);
4996 "cmpxchg survived AtomicExpand at optlevel > -O0");
4998 auto *RetPairTy = cast<StructType>(
I->getType());
4999 Type *
RetTy = RetPairTy->getTypeAtIndex(0U);
5000 assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) &&
5001 "cmpxchg has a non-i1 status result");
5004 if (!isTypeLegal(
RetTy, VT))
5008 unsigned Opc, CmpOpc;
5012 Opc = AArch64::CMP_SWAP_32;
5013 CmpOpc = AArch64::SUBSWrs;
5014 ResRC = &AArch64::GPR32RegClass;
5016 Opc = AArch64::CMP_SWAP_64;
5017 CmpOpc = AArch64::SUBSXrs;
5018 ResRC = &AArch64::GPR64RegClass;
5026 II, getRegForValue(
I->getPointerOperand()), II.
getNumDefs());
5028 II, getRegForValue(
I->getCompareOperand()), II.
getNumDefs() + 1);
5030 II, getRegForValue(
I->getNewValOperand()), II.
getNumDefs() + 2);
5032 const Register ResultReg1 = createResultReg(ResRC);
5033 const Register ResultReg2 = createResultReg(&AArch64::GPR32RegClass);
5034 const Register ScratchReg = createResultReg(&AArch64::GPR32RegClass);
5037 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
5044 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(CmpOpc))
5050 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,