57#include "llvm/IR/IntrinsicsAArch64.h"
81class AArch64FastISel final :
public FastISel {
84 using BaseKind =
enum {
90 BaseKind
Kind = RegBase;
96 unsigned OffsetReg = 0;
104 void setKind(BaseKind K) {
Kind =
K; }
105 BaseKind getKind()
const {
return Kind; }
108 bool isRegBase()
const {
return Kind == RegBase; }
109 bool isFIBase()
const {
return Kind == FrameIndexBase; }
111 void setReg(
unsigned Reg) {
112 assert(isRegBase() &&
"Invalid base register access!");
117 assert(isRegBase() &&
"Invalid base register access!");
121 void setOffsetReg(
unsigned Reg) {
125 unsigned getOffsetReg()
const {
129 void setFI(
unsigned FI) {
130 assert(isFIBase() &&
"Invalid base frame index access!");
134 unsigned getFI()
const {
135 assert(isFIBase() &&
"Invalid base frame index access!");
139 void setOffset(int64_t O) {
Offset =
O; }
141 void setShift(
unsigned S) { Shift = S; }
142 unsigned getShift() {
return Shift; }
171 bool selectRem(
const Instruction *
I,
unsigned ISDOpcode);
184 bool isTypeLegal(
Type *Ty,
MVT &VT);
185 bool isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed =
false);
186 bool isValueAvailable(
const Value *V)
const;
187 bool computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty =
nullptr);
188 bool computeCallAddress(
const Value *V, Address &
Addr);
189 bool simplifyAddress(Address &
Addr,
MVT VT);
194 bool tryEmitSmallMemCpy(Address Dest, Address Src,
uint64_t Len,
203 unsigned emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
204 const Value *RHS,
bool SetFlags =
false,
205 bool WantResult =
true,
bool IsZExt =
false);
206 unsigned emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
207 unsigned RHSReg,
bool SetFlags =
false,
208 bool WantResult =
true);
209 unsigned emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
210 uint64_t Imm,
bool SetFlags =
false,
211 bool WantResult =
true);
212 unsigned emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
214 uint64_t ShiftImm,
bool SetFlags =
false,
215 bool WantResult =
true);
216 unsigned emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
218 uint64_t ShiftImm,
bool SetFlags =
false,
219 bool WantResult =
true);
222 bool emitCompareAndBranch(
const BranchInst *BI);
224 bool emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
bool IsZExt);
225 bool emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
226 bool emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS);
231 bool emitStoreRelease(
MVT VT,
unsigned SrcReg,
unsigned AddrReg,
233 unsigned emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
bool isZExt);
234 unsigned emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool isZExt);
235 unsigned emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
236 bool SetFlags =
false,
bool WantResult =
true,
237 bool IsZExt =
false);
238 unsigned emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm);
239 unsigned emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
240 bool SetFlags =
false,
bool WantResult =
true,
241 bool IsZExt =
false);
242 unsigned emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
243 bool WantResult =
true);
244 unsigned emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
246 bool WantResult =
true);
247 unsigned emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
const Value *LHS,
249 unsigned emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
251 unsigned emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
252 unsigned RHSReg,
uint64_t ShiftImm);
253 unsigned emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
254 unsigned emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
255 unsigned emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
256 unsigned emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
257 unsigned emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
258 unsigned emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
260 unsigned emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
261 unsigned emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
263 unsigned emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
264 unsigned emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
265 bool IsZExt =
false);
276 bool finishCall(CallLoweringInfo &CLI,
unsigned NumBytes);
293#include "AArch64GenFastISel.inc"
300 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
301 "Unexpected integer extend instruction.");
302 assert(!
I->getType()->isVectorTy() &&
I->getType()->isIntegerTy() &&
303 "Unexpected value type.");
304 bool IsZExt = isa<ZExtInst>(
I);
306 if (
const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0)))
310 if (
const auto *Arg = dyn_cast<Argument>(
I->getOperand(0)))
311 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr()))
342 if (Subtarget->isTargetDarwin())
344 if (Subtarget->isTargetWindows())
349unsigned AArch64FastISel::fastMaterializeAlloca(
const AllocaInst *AI) {
351 "Alloca should always return a pointer.");
354 if (!FuncInfo.StaticAllocaMap.count(AI))
358 FuncInfo.StaticAllocaMap.find(AI);
360 if (SI != FuncInfo.StaticAllocaMap.end()) {
361 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
362 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
373unsigned AArch64FastISel::materializeInt(
const ConstantInt *CI,
MVT VT) {
382 : &AArch64::GPR32RegClass;
383 unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
384 Register ResultReg = createResultReg(RC);
385 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(TargetOpcode::COPY),
390unsigned AArch64FastISel::materializeFP(
const ConstantFP *CFP,
MVT VT) {
394 return fastMaterializeFloatZero(CFP);
396 if (VT != MVT::f32 && VT != MVT::f64)
400 bool Is64Bit = (VT == MVT::f64);
406 unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
407 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
412 unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm;
414 &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
416 Register TmpReg = createResultReg(RC);
417 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc1), TmpReg)
420 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
421 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
422 TII.get(TargetOpcode::COPY), ResultReg)
432 unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Alignment);
433 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
434 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
437 unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui;
438 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
439 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
445unsigned AArch64FastISel::materializeGV(
const GlobalValue *GV) {
452 if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
455 unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
457 EVT DestEVT = TLI.getValueType(
DL, GV->
getType(),
true);
461 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
466 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
471 if (Subtarget->isTargetILP32()) {
472 ResultReg = createResultReg(&AArch64::GPR32RegClass);
473 LdrOpc = AArch64::LDRWui;
475 ResultReg = createResultReg(&AArch64::GPR64RegClass);
476 LdrOpc = AArch64::LDRXui;
478 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(LdrOpc),
483 if (!Subtarget->isTargetILP32())
488 Register Result64 = createResultReg(&AArch64::GPR64RegClass);
489 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
490 TII.get(TargetOpcode::SUBREG_TO_REG))
498 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
516 unsigned DstReg = createResultReg(&AArch64::GPR64commonRegClass);
517 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::MOVKXi),
526 ResultReg = createResultReg(&AArch64::GPR64spRegClass);
527 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
537unsigned AArch64FastISel::fastMaterializeConstant(
const Constant *
C) {
538 EVT CEVT = TLI.getValueType(
DL,
C->getType(),
true);
546 if (isa<ConstantPointerNull>(
C)) {
547 assert(VT == MVT::i64 &&
"Expected 64-bit pointers");
551 if (
const auto *CI = dyn_cast<ConstantInt>(
C))
552 return materializeInt(CI, VT);
553 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
554 return materializeFP(CFP, VT);
555 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(
C))
556 return materializeGV(GV);
561unsigned AArch64FastISel::fastMaterializeFloatZero(
const ConstantFP* CFP) {
563 "Floating-point constant is not a positive zero.");
565 if (!isTypeLegal(CFP->
getType(), VT))
568 if (VT != MVT::f32 && VT != MVT::f64)
571 bool Is64Bit = (VT == MVT::f64);
572 unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
573 unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr;
574 return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg);
579 if (
const auto *
MI = dyn_cast<MulOperator>(
I)) {
580 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(0)))
581 if (
C->getValue().isPowerOf2())
583 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(1)))
584 if (
C->getValue().isPowerOf2())
591bool AArch64FastISel::computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty)
593 const User *
U =
nullptr;
594 unsigned Opcode = Instruction::UserOp1;
595 if (
const Instruction *
I = dyn_cast<Instruction>(Obj)) {
598 if (FuncInfo.StaticAllocaMap.count(
static_cast<const AllocaInst *
>(Obj)) ||
599 FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
600 Opcode =
I->getOpcode();
603 }
else if (
const ConstantExpr *
C = dyn_cast<ConstantExpr>(Obj)) {
604 Opcode =
C->getOpcode();
608 if (
auto *Ty = dyn_cast<PointerType>(Obj->
getType()))
609 if (Ty->getAddressSpace() > 255)
617 case Instruction::BitCast:
619 return computeAddress(
U->getOperand(0),
Addr, Ty);
621 case Instruction::IntToPtr:
623 if (TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
624 TLI.getPointerTy(
DL))
625 return computeAddress(
U->getOperand(0),
Addr, Ty);
628 case Instruction::PtrToInt:
630 if (TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
631 return computeAddress(
U->getOperand(0),
Addr, Ty);
634 case Instruction::GetElementPtr: {
642 const Value *
Op = GTI.getOperand();
643 if (
StructType *STy = GTI.getStructTypeOrNull()) {
645 unsigned Idx = cast<ConstantInt>(
Op)->getZExtValue();
648 uint64_t S = GTI.getSequentialElementStride(
DL);
655 if (canFoldAddIntoGEP(U,
Op)) {
658 cast<ConstantInt>(cast<AddOperator>(
Op)->getOperand(1));
661 Op = cast<AddOperator>(
Op)->getOperand(0);
665 goto unsupported_gep;
671 Addr.setOffset(TmpOffset);
672 if (computeAddress(
U->getOperand(0),
Addr, Ty))
681 case Instruction::Alloca: {
684 FuncInfo.StaticAllocaMap.find(AI);
685 if (SI != FuncInfo.StaticAllocaMap.end()) {
686 Addr.setKind(Address::FrameIndexBase);
692 case Instruction::Add: {
697 if (isa<ConstantInt>(LHS))
700 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
702 return computeAddress(LHS,
Addr, Ty);
706 if (computeAddress(LHS,
Addr, Ty) && computeAddress(RHS,
Addr, Ty))
712 case Instruction::Sub: {
717 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
719 return computeAddress(LHS,
Addr, Ty);
723 case Instruction::Shl: {
724 if (
Addr.getOffsetReg())
727 const auto *CI = dyn_cast<ConstantInt>(
U->getOperand(1));
732 if (Val < 1 || Val > 3)
738 NumBytes = NumBits / 8;
743 if (NumBytes != (1ULL << Val))
749 const Value *Src =
U->getOperand(0);
750 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
751 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
753 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
755 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
757 Src = ZE->getOperand(0);
759 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
761 SE->getOperand(0)->getType()->isIntegerTy(32)) {
763 Src = SE->getOperand(0);
769 if (
const auto *AI = dyn_cast<BinaryOperator>(Src))
770 if (AI->
getOpcode() == Instruction::And) {
774 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
775 if (
C->getValue() == 0xffffffff)
778 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
779 if (
C->getValue() == 0xffffffff) {
784 Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);
785 Addr.setOffsetReg(Reg);
793 Addr.setOffsetReg(Reg);
796 case Instruction::Mul: {
797 if (
Addr.getOffsetReg())
807 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
808 if (
C->getValue().isPowerOf2())
811 assert(isa<ConstantInt>(RHS) &&
"Expected an ConstantInt.");
812 const auto *
C = cast<ConstantInt>(RHS);
813 unsigned Val =
C->getValue().logBase2();
814 if (Val < 1 || Val > 3)
820 NumBytes = NumBits / 8;
825 if (NumBytes != (1ULL << Val))
832 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
833 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
835 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
837 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
839 Src = ZE->getOperand(0);
841 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
843 SE->getOperand(0)->getType()->isIntegerTy(32)) {
845 Src = SE->getOperand(0);
854 Addr.setOffsetReg(Reg);
857 case Instruction::And: {
858 if (
Addr.getOffsetReg())
861 if (!Ty ||
DL.getTypeSizeInBits(Ty) != 8)
867 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
868 if (
C->getValue() == 0xffffffff)
871 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
872 if (
C->getValue() == 0xffffffff) {
880 Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);
881 Addr.setOffsetReg(Reg);
886 case Instruction::SExt:
887 case Instruction::ZExt: {
888 if (!
Addr.getReg() ||
Addr.getOffsetReg())
891 const Value *Src =
nullptr;
893 if (
const auto *ZE = dyn_cast<ZExtInst>(U)) {
894 if (!
isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
896 Src = ZE->getOperand(0);
898 }
else if (
const auto *SE = dyn_cast<SExtInst>(U)) {
899 if (!
isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
901 Src = SE->getOperand(0);
912 Addr.setOffsetReg(Reg);
917 if (
Addr.isRegBase() && !
Addr.getReg()) {
925 if (!
Addr.getOffsetReg()) {
929 Addr.setOffsetReg(Reg);
936bool AArch64FastISel::computeCallAddress(
const Value *V, Address &
Addr) {
937 const User *
U =
nullptr;
938 unsigned Opcode = Instruction::UserOp1;
941 if (
const auto *
I = dyn_cast<Instruction>(V)) {
942 Opcode =
I->getOpcode();
944 InMBB =
I->getParent() == FuncInfo.MBB->getBasicBlock();
945 }
else if (
const auto *
C = dyn_cast<ConstantExpr>(V)) {
946 Opcode =
C->getOpcode();
952 case Instruction::BitCast:
955 return computeCallAddress(
U->getOperand(0),
Addr);
957 case Instruction::IntToPtr:
960 TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
961 TLI.getPointerTy(
DL))
962 return computeCallAddress(
U->getOperand(0),
Addr);
964 case Instruction::PtrToInt:
966 if (InMBB && TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
967 return computeCallAddress(
U->getOperand(0),
Addr);
971 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
972 Addr.setGlobalValue(GV);
977 if (!
Addr.getGlobalValue()) {
978 Addr.setReg(getRegForValue(V));
979 return Addr.getReg() != 0;
985bool AArch64FastISel::isTypeLegal(
Type *Ty,
MVT &VT) {
986 EVT evt = TLI.getValueType(
DL, Ty,
true);
988 if (Subtarget->isTargetILP32() && Ty->
isPointerTy())
992 if (evt == MVT::Other || !evt.
isSimple())
1002 return TLI.isTypeLegal(VT);
1009bool AArch64FastISel::isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed) {
1013 if (isTypeLegal(Ty, VT))
1018 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
1024bool AArch64FastISel::isValueAvailable(
const Value *V)
const {
1025 if (!isa<Instruction>(V))
1028 const auto *
I = cast<Instruction>(V);
1029 return FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB;
1032bool AArch64FastISel::simplifyAddress(Address &
Addr,
MVT VT) {
1033 if (Subtarget->isTargetILP32())
1040 bool ImmediateOffsetNeedsLowering =
false;
1041 bool RegisterOffsetNeedsLowering =
false;
1044 ImmediateOffsetNeedsLowering =
true;
1045 else if (
Offset > 0 && !(
Offset & (ScaleFactor - 1)) &&
1046 !isUInt<12>(
Offset / ScaleFactor))
1047 ImmediateOffsetNeedsLowering =
true;
1052 if (!ImmediateOffsetNeedsLowering &&
Addr.getOffset() &&
Addr.getOffsetReg())
1053 RegisterOffsetNeedsLowering =
true;
1056 if (
Addr.isRegBase() &&
Addr.getOffsetReg() && !
Addr.getReg())
1057 RegisterOffsetNeedsLowering =
true;
1062 if ((ImmediateOffsetNeedsLowering ||
Addr.getOffsetReg()) &&
Addr.isFIBase())
1064 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
1065 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
1070 Addr.setKind(Address::RegBase);
1071 Addr.setReg(ResultReg);
1074 if (RegisterOffsetNeedsLowering) {
1075 unsigned ResultReg = 0;
1076 if (
Addr.getReg()) {
1079 ResultReg = emitAddSub_rx(
true, MVT::i64,
Addr.getReg(),
1080 Addr.getOffsetReg(),
Addr.getExtendType(),
1083 ResultReg = emitAddSub_rs(
true, MVT::i64,
Addr.getReg(),
1088 ResultReg = emitLSL_ri(MVT::i64, MVT::i32,
Addr.getOffsetReg(),
1089 Addr.getShift(),
true);
1091 ResultReg = emitLSL_ri(MVT::i64, MVT::i32,
Addr.getOffsetReg(),
1092 Addr.getShift(),
false);
1094 ResultReg = emitLSL_ri(MVT::i64, MVT::i64,
Addr.getOffsetReg(),
1100 Addr.setReg(ResultReg);
1101 Addr.setOffsetReg(0);
1108 if (ImmediateOffsetNeedsLowering) {
1112 ResultReg = emitAdd_ri_(MVT::i64,
Addr.getReg(),
Offset);
1118 Addr.setReg(ResultReg);
1124void AArch64FastISel::addLoadStoreOperands(Address &
Addr,
1127 unsigned ScaleFactor,
1129 int64_t
Offset =
Addr.getOffset() / ScaleFactor;
1131 if (
Addr.isFIBase()) {
1132 int FI =
Addr.getFI();
1135 MMO = FuncInfo.MF->getMachineMemOperand(
1137 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
1141 assert(
Addr.isRegBase() &&
"Unexpected address kind.");
1148 if (
Addr.getOffsetReg()) {
1149 assert(
Addr.getOffset() == 0 &&
"Unexpected offset");
1164unsigned AArch64FastISel::emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
1165 const Value *RHS,
bool SetFlags,
1166 bool WantResult,
bool IsZExt) {
1168 bool NeedExtend =
false;
1191 if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
1195 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1200 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1201 if (
const auto *SI = dyn_cast<BinaryOperator>(LHS))
1202 if (isa<ConstantInt>(
SI->getOperand(1)))
1203 if (
SI->getOpcode() == Instruction::Shl ||
1204 SI->getOpcode() == Instruction::LShr ||
1205 SI->getOpcode() == Instruction::AShr )
1208 Register LHSReg = getRegForValue(LHS);
1213 LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
1215 unsigned ResultReg = 0;
1216 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1217 uint64_t Imm = IsZExt ?
C->getZExtValue() :
C->getSExtValue();
1218 if (
C->isNegative())
1219 ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,
1222 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,
1224 }
else if (
const auto *
C = dyn_cast<Constant>(RHS))
1225 if (
C->isNullValue())
1226 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);
1233 isValueAvailable(RHS)) {
1234 Register RHSReg = getRegForValue(RHS);
1237 return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,
1238 SetFlags, WantResult);
1244 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1245 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1247 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1248 if (
C->getValue().isPowerOf2())
1251 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1252 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1253 Register RHSReg = getRegForValue(MulLHS);
1256 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg,
AArch64_AM::LSL,
1257 ShiftVal, SetFlags, WantResult);
1265 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
1266 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1268 switch (
SI->getOpcode()) {
1276 Register RHSReg = getRegForValue(
SI->getOperand(0));
1279 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,
1280 ShiftVal, SetFlags, WantResult);
1288 Register RHSReg = getRegForValue(RHS);
1293 RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
1295 return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);
1298unsigned AArch64FastISel::emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1299 unsigned RHSReg,
bool SetFlags,
1301 assert(LHSReg && RHSReg &&
"Invalid register number.");
1303 if (LHSReg == AArch64::SP || LHSReg == AArch64::WSP ||
1304 RHSReg == AArch64::SP || RHSReg == AArch64::WSP)
1307 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1310 static const unsigned OpcTable[2][2][2] = {
1311 { { AArch64::SUBWrr, AArch64::SUBXrr },
1312 { AArch64::ADDWrr, AArch64::ADDXrr } },
1313 { { AArch64::SUBSWrr, AArch64::SUBSXrr },
1314 { AArch64::ADDSWrr, AArch64::ADDSXrr } }
1316 bool Is64Bit = RetVT == MVT::i64;
1317 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1319 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1322 ResultReg = createResultReg(RC);
1324 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1329 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1335unsigned AArch64FastISel::emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1338 assert(LHSReg &&
"Invalid register number.");
1340 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1344 if (isUInt<12>(Imm))
1346 else if ((Imm & 0xfff000) == Imm) {
1352 static const unsigned OpcTable[2][2][2] = {
1353 { { AArch64::SUBWri, AArch64::SUBXri },
1354 { AArch64::ADDWri, AArch64::ADDXri } },
1355 { { AArch64::SUBSWri, AArch64::SUBSXri },
1356 { AArch64::ADDSWri, AArch64::ADDSXri } }
1358 bool Is64Bit = RetVT == MVT::i64;
1359 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1362 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1364 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1367 ResultReg = createResultReg(RC);
1369 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1373 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1380unsigned AArch64FastISel::emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1385 assert(LHSReg && RHSReg &&
"Invalid register number.");
1386 assert(LHSReg != AArch64::SP && LHSReg != AArch64::WSP &&
1387 RHSReg != AArch64::SP && RHSReg != AArch64::WSP);
1389 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1396 static const unsigned OpcTable[2][2][2] = {
1397 { { AArch64::SUBWrs, AArch64::SUBXrs },
1398 { AArch64::ADDWrs, AArch64::ADDXrs } },
1399 { { AArch64::SUBSWrs, AArch64::SUBSXrs },
1400 { AArch64::ADDSWrs, AArch64::ADDSXrs } }
1402 bool Is64Bit = RetVT == MVT::i64;
1403 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1405 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1408 ResultReg = createResultReg(RC);
1410 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1415 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1418 .
addImm(getShifterImm(ShiftType, ShiftImm));
1422unsigned AArch64FastISel::emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1427 assert(LHSReg && RHSReg &&
"Invalid register number.");
1428 assert(LHSReg != AArch64::XZR && LHSReg != AArch64::WZR &&
1429 RHSReg != AArch64::XZR && RHSReg != AArch64::WZR);
1431 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1437 static const unsigned OpcTable[2][2][2] = {
1438 { { AArch64::SUBWrx, AArch64::SUBXrx },
1439 { AArch64::ADDWrx, AArch64::ADDXrx } },
1440 { { AArch64::SUBSWrx, AArch64::SUBSXrx },
1441 { AArch64::ADDSWrx, AArch64::ADDSXrx } }
1443 bool Is64Bit = RetVT == MVT::i64;
1444 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1447 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1449 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1452 ResultReg = createResultReg(RC);
1454 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1459 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1462 .
addImm(getArithExtendImm(ExtType, ShiftImm));
1466bool AArch64FastISel::emitCmp(
const Value *LHS,
const Value *RHS,
bool IsZExt) {
1468 EVT EVT = TLI.getValueType(
DL, Ty,
true);
1481 return emitICmp(VT, LHS, RHS, IsZExt);
1484 return emitFCmp(VT, LHS, RHS);
1488bool AArch64FastISel::emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1490 return emitSub(RetVT, LHS, RHS,
true,
false,
1494bool AArch64FastISel::emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm) {
1495 return emitAddSub_ri(
false, RetVT, LHSReg, Imm,
1499bool AArch64FastISel::emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS) {
1500 if (RetVT != MVT::f32 && RetVT != MVT::f64)
1505 bool UseImm =
false;
1506 if (
const auto *CFP = dyn_cast<ConstantFP>(RHS))
1510 Register LHSReg = getRegForValue(LHS);
1515 unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;
1516 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1521 Register RHSReg = getRegForValue(RHS);
1525 unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;
1526 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1532unsigned AArch64FastISel::emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1533 bool SetFlags,
bool WantResult,
bool IsZExt) {
1534 return emitAddSub(
true, RetVT, LHS, RHS, SetFlags, WantResult,
1543unsigned AArch64FastISel::emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm) {
1546 ResultReg = emitAddSub_ri(
false, VT, Op0, -Imm);
1548 ResultReg = emitAddSub_ri(
true, VT, Op0, Imm);
1557 ResultReg = emitAddSub_rr(
true, VT, Op0, CReg);
1561unsigned AArch64FastISel::emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1562 bool SetFlags,
bool WantResult,
bool IsZExt) {
1563 return emitAddSub(
false, RetVT, LHS, RHS, SetFlags, WantResult,
1567unsigned AArch64FastISel::emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
1568 unsigned RHSReg,
bool WantResult) {
1569 return emitAddSub_rr(
false, RetVT, LHSReg, RHSReg,
1573unsigned AArch64FastISel::emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
1576 uint64_t ShiftImm,
bool WantResult) {
1577 return emitAddSub_rs(
false, RetVT, LHSReg, RHSReg, ShiftType,
1578 ShiftImm,
true, WantResult);
1581unsigned AArch64FastISel::emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
1584 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
1594 if (
const auto *SI = dyn_cast<ShlOperator>(LHS))
1595 if (isa<ConstantInt>(
SI->getOperand(1)))
1598 Register LHSReg = getRegForValue(LHS);
1602 unsigned ResultReg = 0;
1603 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1605 ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);
1613 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1614 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1616 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1617 if (
C->getValue().isPowerOf2())
1620 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1621 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1623 Register RHSReg = getRegForValue(MulLHS);
1626 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1634 if (
const auto *SI = dyn_cast<ShlOperator>(RHS))
1635 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1637 Register RHSReg = getRegForValue(
SI->getOperand(0));
1640 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1646 Register RHSReg = getRegForValue(RHS);
1651 ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, RHSReg);
1652 if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {
1654 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1659unsigned AArch64FastISel::emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
1662 "ISD nodes are not consecutive!");
1663 static const unsigned OpcTable[3][2] = {
1664 { AArch64::ANDWri, AArch64::ANDXri },
1665 { AArch64::ORRWri, AArch64::ORRXri },
1666 { AArch64::EORWri, AArch64::EORXri }
1679 Opc = OpcTable[
Idx][0];
1680 RC = &AArch64::GPR32spRegClass;
1685 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1686 RC = &AArch64::GPR64spRegClass;
1695 fastEmitInst_ri(Opc, RC, LHSReg,
1697 if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc !=
ISD::AND) {
1699 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1704unsigned AArch64FastISel::emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
1705 unsigned LHSReg,
unsigned RHSReg,
1708 "ISD nodes are not consecutive!");
1709 static const unsigned OpcTable[3][2] = {
1710 { AArch64::ANDWrs, AArch64::ANDXrs },
1711 { AArch64::ORRWrs, AArch64::ORRXrs },
1712 { AArch64::EORWrs, AArch64::EORXrs }
1728 Opc = OpcTable[ISDOpc -
ISD::AND][0];
1729 RC = &AArch64::GPR32RegClass;
1732 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1733 RC = &AArch64::GPR64RegClass;
1737 fastEmitInst_rri(Opc, RC, LHSReg, RHSReg,
1739 if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {
1741 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1746unsigned AArch64FastISel::emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
1748 return emitLogicalOp_ri(
ISD::AND, RetVT, LHSReg, Imm);
1751unsigned AArch64FastISel::emitLoad(
MVT VT,
MVT RetVT, Address
Addr,
1753 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1757 if (!simplifyAddress(
Addr, VT))
1766 bool UseScaled =
true;
1767 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
1772 static const unsigned GPOpcTable[2][8][4] = {
1774 { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi,
1776 { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi,
1778 { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui,
1780 { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui,
1782 { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX,
1784 { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX,
1786 { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW,
1788 { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW,
1792 { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1794 { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1796 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1798 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1800 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1802 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1804 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1806 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1811 static const unsigned FPOpcTable[4][2] = {
1812 { AArch64::LDURSi, AArch64::LDURDi },
1813 { AArch64::LDRSui, AArch64::LDRDui },
1814 { AArch64::LDRSroX, AArch64::LDRDroX },
1815 { AArch64::LDRSroW, AArch64::LDRDroW }
1820 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
1821 Addr.getOffsetReg();
1822 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
1827 bool IsRet64Bit = RetVT == MVT::i64;
1833 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][0];
1834 RC = (IsRet64Bit && !WantZExt) ?
1835 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1838 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][1];
1839 RC = (IsRet64Bit && !WantZExt) ?
1840 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1843 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][2];
1844 RC = (IsRet64Bit && !WantZExt) ?
1845 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1848 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][3];
1849 RC = &AArch64::GPR64RegClass;
1852 Opc = FPOpcTable[
Idx][0];
1853 RC = &AArch64::FPR32RegClass;
1856 Opc = FPOpcTable[
Idx][1];
1857 RC = &AArch64::FPR64RegClass;
1862 Register ResultReg = createResultReg(RC);
1864 TII.get(Opc), ResultReg);
1868 if (VT == MVT::i1) {
1869 unsigned ANDReg = emitAnd_ri(MVT::i32, ResultReg, 1);
1870 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
1876 if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {
1877 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
1878 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1879 TII.get(AArch64::SUBREG_TO_REG), Reg64)
1882 .
addImm(AArch64::sub_32);
1888bool AArch64FastISel::selectAddSub(
const Instruction *
I) {
1890 if (!isTypeSupported(
I->getType(), VT,
true))
1894 return selectOperator(
I,
I->getOpcode());
1897 switch (
I->getOpcode()) {
1900 case Instruction::Add:
1901 ResultReg = emitAdd(VT,
I->getOperand(0),
I->getOperand(1));
1903 case Instruction::Sub:
1904 ResultReg = emitSub(VT,
I->getOperand(0),
I->getOperand(1));
1910 updateValueMap(
I, ResultReg);
1914bool AArch64FastISel::selectLogicalOp(
const Instruction *
I) {
1916 if (!isTypeSupported(
I->getType(), VT,
true))
1920 return selectOperator(
I,
I->getOpcode());
1923 switch (
I->getOpcode()) {
1926 case Instruction::And:
1927 ResultReg = emitLogicalOp(
ISD::AND, VT,
I->getOperand(0),
I->getOperand(1));
1929 case Instruction::Or:
1930 ResultReg = emitLogicalOp(
ISD::OR, VT,
I->getOperand(0),
I->getOperand(1));
1932 case Instruction::Xor:
1933 ResultReg = emitLogicalOp(
ISD::XOR, VT,
I->getOperand(0),
I->getOperand(1));
1939 updateValueMap(
I, ResultReg);
1943bool AArch64FastISel::selectLoad(
const Instruction *
I) {
1948 if (!isTypeSupported(
I->getType(), VT,
true) ||
1949 cast<LoadInst>(
I)->isAtomic())
1952 const Value *SV =
I->getOperand(0);
1953 if (TLI.supportSwiftError()) {
1956 if (
const Argument *Arg = dyn_cast<Argument>(SV)) {
1957 if (Arg->hasSwiftErrorAttr())
1961 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
1962 if (Alloca->isSwiftError())
1969 if (!computeAddress(
I->getOperand(0),
Addr,
I->getType()))
1973 bool WantZExt =
true;
1975 const Value *IntExtVal =
nullptr;
1976 if (
I->hasOneUse()) {
1977 if (
const auto *ZE = dyn_cast<ZExtInst>(
I->use_begin()->getUser())) {
1978 if (isTypeSupported(ZE->getType(), RetVT))
1982 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I->use_begin()->getUser())) {
1983 if (isTypeSupported(SE->getType(), RetVT))
1991 unsigned ResultReg =
1992 emitLoad(VT, RetVT,
Addr, WantZExt, createMachineMemOperandFor(
I));
2013 auto *
MI =
MRI.getUniqueVRegDef(Reg);
2015 if (RetVT == MVT::i64 && VT <= MVT::i32) {
2019 ResultReg = std::prev(
I)->getOperand(0).getReg();
2020 removeDeadCode(
I, std::next(
I));
2022 ResultReg = fastEmitInst_extractsubreg(MVT::i32, ResultReg,
2025 updateValueMap(
I, ResultReg);
2034 for (
auto &Opnd :
MI->uses()) {
2036 Reg = Opnd.getReg();
2041 removeDeadCode(
I, std::next(
I));
2044 MI =
MRI.getUniqueVRegDef(Reg);
2046 updateValueMap(IntExtVal, ResultReg);
2050 updateValueMap(
I, ResultReg);
2054bool AArch64FastISel::emitStoreRelease(
MVT VT,
unsigned SrcReg,
2059 default:
return false;
2060 case MVT::i8: Opc = AArch64::STLRB;
break;
2061 case MVT::i16: Opc = AArch64::STLRH;
break;
2062 case MVT::i32: Opc = AArch64::STLRW;
break;
2063 case MVT::i64: Opc = AArch64::STLRX;
break;
2069 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2076bool AArch64FastISel::emitStore(
MVT VT,
unsigned SrcReg, Address
Addr,
2078 if (!TLI.allowsMisalignedMemoryAccesses(VT))
2082 if (!simplifyAddress(
Addr, VT))
2091 bool UseScaled =
true;
2092 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
2097 static const unsigned OpcTable[4][6] = {
2098 { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi,
2099 AArch64::STURSi, AArch64::STURDi },
2100 { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui,
2101 AArch64::STRSui, AArch64::STRDui },
2102 { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX,
2103 AArch64::STRSroX, AArch64::STRDroX },
2104 { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW,
2105 AArch64::STRSroW, AArch64::STRDroW }
2109 bool VTIsi1 =
false;
2110 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
2111 Addr.getOffsetReg();
2112 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
2119 case MVT::i1: VTIsi1 =
true; [[fallthrough]];
2120 case MVT::i8: Opc = OpcTable[
Idx][0];
break;
2121 case MVT::i16: Opc = OpcTable[
Idx][1];
break;
2122 case MVT::i32: Opc = OpcTable[
Idx][2];
break;
2123 case MVT::i64: Opc = OpcTable[
Idx][3];
break;
2124 case MVT::f32: Opc = OpcTable[
Idx][4];
break;
2125 case MVT::f64: Opc = OpcTable[
Idx][5];
break;
2129 if (VTIsi1 && SrcReg != AArch64::WZR) {
2130 unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, 1);
2131 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2138 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(SrcReg);
2144bool AArch64FastISel::selectStore(
const Instruction *
I) {
2146 const Value *Op0 =
I->getOperand(0);
2150 if (!isTypeSupported(Op0->
getType(), VT,
true))
2153 const Value *PtrV =
I->getOperand(1);
2154 if (TLI.supportSwiftError()) {
2157 if (
const Argument *Arg = dyn_cast<Argument>(PtrV)) {
2158 if (Arg->hasSwiftErrorAttr())
2162 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
2163 if (Alloca->isSwiftError())
2170 unsigned SrcReg = 0;
2171 if (
const auto *CI = dyn_cast<ConstantInt>(Op0)) {
2173 SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
2174 }
else if (
const auto *CF = dyn_cast<ConstantFP>(Op0)) {
2175 if (CF->isZero() && !CF->isNegative()) {
2177 SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
2182 SrcReg = getRegForValue(Op0);
2187 auto *
SI = cast<StoreInst>(
I);
2190 if (
SI->isAtomic()) {
2195 Register AddrReg = getRegForValue(PtrV);
2196 return emitStoreRelease(VT, SrcReg, AddrReg,
2197 createMachineMemOperandFor(
I));
2258bool AArch64FastISel::emitCompareAndBranch(
const BranchInst *BI) {
2262 if (FuncInfo.MF->getFunction().hasFnAttribute(
2263 Attribute::SpeculativeLoadHardening))
2285 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2292 switch (Predicate) {
2297 if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue())
2300 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2303 if (
const auto *AI = dyn_cast<BinaryOperator>(LHS))
2304 if (AI->
getOpcode() == Instruction::And && isValueAvailable(AI)) {
2308 if (
const auto *
C = dyn_cast<ConstantInt>(AndLHS))
2309 if (
C->getValue().isPowerOf2())
2312 if (
const auto *
C = dyn_cast<ConstantInt>(AndRHS))
2313 if (
C->getValue().isPowerOf2()) {
2314 TestBit =
C->getValue().logBase2();
2326 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2334 if (!isa<ConstantInt>(RHS))
2337 if (cast<ConstantInt>(RHS)->getValue() !=
APInt(BW, -1,
true))
2345 static const unsigned OpcTable[2][2][2] = {
2346 { {AArch64::CBZW, AArch64::CBZX },
2347 {AArch64::CBNZW, AArch64::CBNZX} },
2348 { {AArch64::TBZW, AArch64::TBZX },
2349 {AArch64::TBNZW, AArch64::TBNZX} }
2352 bool IsBitTest = TestBit != -1;
2353 bool Is64Bit = BW == 64;
2354 if (TestBit < 32 && TestBit >= 0)
2357 unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
2360 Register SrcReg = getRegForValue(LHS);
2364 if (BW == 64 && !Is64Bit)
2365 SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, AArch64::sub_32);
2367 if ((BW < 32) && !IsBitTest)
2368 SrcReg = emitIntExt(VT, SrcReg, MVT::i32,
true);
2373 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
2383bool AArch64FastISel::selectBranch(
const Instruction *
I) {
2395 if (CI->
hasOneUse() && isValueAvailable(CI)) {
2398 switch (Predicate) {
2402 fastEmitBranch(FBB, MIMD.getDL());
2405 fastEmitBranch(
TBB, MIMD.getDL());
2410 if (emitCompareAndBranch(BI))
2414 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2427 switch (Predicate) {
2443 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2449 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2456 }
else if (
const auto *CI = dyn_cast<ConstantInt>(BI->
getCondition())) {
2459 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::B))
2468 FuncInfo.MBB->addSuccessorWithoutProb(
Target);
2480 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2494 unsigned Opcode = AArch64::TBNZW;
2495 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2497 Opcode = AArch64::TBZW;
2503 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2504 .
addReg(ConstrainedCondReg)
2512bool AArch64FastISel::selectIndirectBr(
const Instruction *
I) {
2521 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(AddrReg);
2525 FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[Succ]);
2531 const CmpInst *CI = cast<CmpInst>(
I);
2539 unsigned ResultReg = 0;
2540 switch (Predicate) {
2544 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2545 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2546 TII.get(TargetOpcode::COPY), ResultReg)
2550 ResultReg = fastEmit_i(MVT::i32, MVT::i32,
ISD::Constant, 1);
2555 updateValueMap(
I, ResultReg);
2563 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2567 static unsigned CondCodeTable[2][2] = {
2572 switch (Predicate) {
2584 Register TmpReg1 = createResultReg(&AArch64::GPR32RegClass);
2585 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2590 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2596 updateValueMap(
I, ResultReg);
2604 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2610 updateValueMap(
I, ResultReg);
2616bool AArch64FastISel::optimizeSelect(
const SelectInst *SI) {
2617 if (!
SI->getType()->isIntegerTy(1))
2620 const Value *Src1Val, *Src2Val;
2622 bool NeedExtraOp =
false;
2623 if (
auto *CI = dyn_cast<ConstantInt>(
SI->getTrueValue())) {
2625 Src1Val =
SI->getCondition();
2626 Src2Val =
SI->getFalseValue();
2627 Opc = AArch64::ORRWrr;
2630 Src1Val =
SI->getFalseValue();
2631 Src2Val =
SI->getCondition();
2632 Opc = AArch64::BICWrr;
2634 }
else if (
auto *CI = dyn_cast<ConstantInt>(
SI->getFalseValue())) {
2636 Src1Val =
SI->getCondition();
2637 Src2Val =
SI->getTrueValue();
2638 Opc = AArch64::ORRWrr;
2642 Src1Val =
SI->getCondition();
2643 Src2Val =
SI->getTrueValue();
2644 Opc = AArch64::ANDWrr;
2651 Register Src1Reg = getRegForValue(Src1Val);
2655 Register Src2Reg = getRegForValue(Src2Val);
2660 Src1Reg = emitLogicalOp_ri(
ISD::XOR, MVT::i32, Src1Reg, 1);
2662 Register ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
2664 updateValueMap(SI, ResultReg);
2668bool AArch64FastISel::selectSelect(
const Instruction *
I) {
2669 assert(isa<SelectInst>(
I) &&
"Expected a select instruction.");
2671 if (!isTypeSupported(
I->getType(), VT))
2683 Opc = AArch64::CSELWr;
2684 RC = &AArch64::GPR32RegClass;
2687 Opc = AArch64::CSELXr;
2688 RC = &AArch64::GPR64RegClass;
2691 Opc = AArch64::FCSELSrrr;
2692 RC = &AArch64::FPR32RegClass;
2695 Opc = AArch64::FCSELDrrr;
2696 RC = &AArch64::FPR64RegClass;
2705 if (optimizeSelect(SI))
2709 if (foldXALUIntrinsic(
CC,
I,
Cond)) {
2714 }
else if (isa<CmpInst>(
Cond) && cast<CmpInst>(
Cond)->hasOneUse() &&
2715 isValueAvailable(
Cond)) {
2716 const auto *
Cmp = cast<CmpInst>(
Cond);
2719 const Value *FoldSelect =
nullptr;
2720 switch (Predicate) {
2724 FoldSelect =
SI->getFalseValue();
2727 FoldSelect =
SI->getTrueValue();
2732 Register SrcReg = getRegForValue(FoldSelect);
2736 updateValueMap(
I, SrcReg);
2746 switch (Predicate) {
2768 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II,
2774 Register Src1Reg = getRegForValue(
SI->getTrueValue());
2775 Register Src2Reg = getRegForValue(
SI->getFalseValue());
2777 if (!Src1Reg || !Src2Reg)
2781 Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg, ExtraCC);
2783 Register ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg,
CC);
2784 updateValueMap(
I, ResultReg);
2788bool AArch64FastISel::selectFPExt(
const Instruction *
I) {
2790 if (!
I->getType()->isDoubleTy() || !
V->getType()->isFloatTy())
2797 Register ResultReg = createResultReg(&AArch64::FPR64RegClass);
2798 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTDSr),
2800 updateValueMap(
I, ResultReg);
2804bool AArch64FastISel::selectFPTrunc(
const Instruction *
I) {
2806 if (!
I->getType()->isFloatTy() || !
V->getType()->isDoubleTy())
2813 Register ResultReg = createResultReg(&AArch64::FPR32RegClass);
2814 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTSDr),
2816 updateValueMap(
I, ResultReg);
2823 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2826 Register SrcReg = getRegForValue(
I->getOperand(0));
2830 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2831 if (SrcVT == MVT::f128 || SrcVT == MVT::f16 || SrcVT == MVT::bf16)
2835 if (SrcVT == MVT::f64) {
2837 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;
2839 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;
2842 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;
2844 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;
2846 Register ResultReg = createResultReg(
2847 DestVT == MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2848 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
2850 updateValueMap(
I, ResultReg);
2856 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2859 if (DestVT == MVT::f16 || DestVT == MVT::bf16)
2862 assert((DestVT == MVT::f32 || DestVT == MVT::f64) &&
2863 "Unexpected value type.");
2865 Register SrcReg = getRegForValue(
I->getOperand(0));
2869 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2872 if (SrcVT == MVT::i16 || SrcVT == MVT::i8 || SrcVT == MVT::i1) {
2880 if (SrcVT == MVT::i64) {
2882 Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;
2884 Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;
2887 Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;
2889 Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;
2892 Register ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg);
2893 updateValueMap(
I, ResultReg);
2897bool AArch64FastISel::fastLowerArguments() {
2898 if (!FuncInfo.CanLowerReturn)
2909 if (Subtarget->hasCustomCallingConv())
2913 unsigned GPRCnt = 0;
2914 unsigned FPRCnt = 0;
2915 for (
auto const &Arg :
F->args()) {
2916 if (Arg.hasAttribute(Attribute::ByVal) ||
2917 Arg.hasAttribute(Attribute::InReg) ||
2918 Arg.hasAttribute(Attribute::StructRet) ||
2919 Arg.hasAttribute(Attribute::SwiftSelf) ||
2920 Arg.hasAttribute(Attribute::SwiftAsync) ||
2921 Arg.hasAttribute(Attribute::SwiftError) ||
2922 Arg.hasAttribute(Attribute::Nest))
2925 Type *ArgTy = Arg.getType();
2929 EVT ArgVT = TLI.getValueType(
DL, ArgTy);
2938 (!Subtarget->hasNEON() || !Subtarget->isLittleEndian()))
2941 if (VT >= MVT::i1 && VT <= MVT::i64)
2943 else if ((VT >= MVT::f16 && VT <= MVT::f64) || VT.
is64BitVector() ||
2949 if (GPRCnt > 8 || FPRCnt > 8)
2954 { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4,
2955 AArch64::W5, AArch64::W6, AArch64::W7 },
2956 { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4,
2957 AArch64::X5, AArch64::X6, AArch64::X7 },
2958 { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4,
2959 AArch64::H5, AArch64::H6, AArch64::H7 },
2960 { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4,
2961 AArch64::S5, AArch64::S6, AArch64::S7 },
2962 { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4,
2963 AArch64::D5, AArch64::D6, AArch64::D7 },
2964 { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4,
2965 AArch64::Q5, AArch64::Q6, AArch64::Q7 }
2969 unsigned FPRIdx = 0;
2970 for (
auto const &Arg :
F->args()) {
2971 MVT VT = TLI.getSimpleValueType(
DL, Arg.getType());
2974 if (VT >= MVT::i1 && VT <= MVT::i32) {
2976 RC = &AArch64::GPR32RegClass;
2978 }
else if (VT == MVT::i64) {
2980 RC = &AArch64::GPR64RegClass;
2981 }
else if (VT == MVT::f16 || VT == MVT::bf16) {
2983 RC = &AArch64::FPR16RegClass;
2984 }
else if (VT == MVT::f32) {
2986 RC = &AArch64::FPR32RegClass;
2989 RC = &AArch64::FPR64RegClass;
2992 RC = &AArch64::FPR128RegClass;
2996 Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
3000 Register ResultReg = createResultReg(RC);
3001 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3002 TII.get(TargetOpcode::COPY), ResultReg)
3004 updateValueMap(&Arg, ResultReg);
3009bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
3011 unsigned &NumBytes) {
3014 CCState CCInfo(
CC,
false, *FuncInfo.MF, ArgLocs, *Context);
3015 CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(
CC));
3018 NumBytes = CCInfo.getStackSize();
3021 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
3022 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackDown))
3027 const Value *ArgVal = CLI.OutVals[VA.getValNo()];
3028 MVT ArgVT = OutVTs[VA.getValNo()];
3030 Register ArgReg = getRegForValue(ArgVal);
3035 switch (VA.getLocInfo()) {
3039 MVT DestVT = VA.getLocVT();
3041 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
false);
3049 MVT DestVT = VA.getLocVT();
3051 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
true);
3061 if (VA.isRegLoc() && !VA.needsCustom()) {
3062 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3063 TII.get(TargetOpcode::COPY), VA.getLocReg()).
addReg(ArgReg);
3064 CLI.OutRegs.push_back(VA.getLocReg());
3065 }
else if (VA.needsCustom()) {
3069 assert(VA.isMemLoc() &&
"Assuming store on stack.");
3072 if (isa<UndefValue>(ArgVal))
3078 unsigned BEAlign = 0;
3079 if (ArgSize < 8 && !Subtarget->isLittleEndian())
3080 BEAlign = 8 - ArgSize;
3083 Addr.setKind(Address::RegBase);
3084 Addr.setReg(AArch64::SP);
3085 Addr.setOffset(VA.getLocMemOffset() + BEAlign);
3099bool AArch64FastISel::finishCall(CallLoweringInfo &CLI,
unsigned NumBytes) {
3103 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3104 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackUp))
3109 CCState CCInfo(
CC,
false, *FuncInfo.MF, RVLocs, *Context);
3110 CCInfo.AnalyzeCallResult(CLI.Ins, CCAssignFnForCall(
CC));
3112 Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3113 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
3116 unsigned CopyReg = ResultReg + i;
3119 if (CopyVT.
isVector() && !Subtarget->isLittleEndian())
3123 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(TargetOpcode::COPY),
3129 CLI.ResultReg = ResultReg;
3130 CLI.NumResultRegs = RVLocs.
size();
3135bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3137 bool IsTailCall = CLI.IsTailCall;
3138 bool IsVarArg = CLI.IsVarArg;
3142 if (!Callee && !Symbol)
3147 if (CLI.CB && CLI.CB->hasFnAttr(Attribute::ReturnsTwice) &&
3148 !Subtarget->noBTIAtReturnTwice() &&
3153 if (CLI.CB && CLI.CB->isIndirectCall() &&
3163 if (Subtarget->isTargetILP32())
3177 if (MF->getFunction().getParent()->getRtLibUseGOT())
3184 if (Subtarget->isWindowsArm64EC())
3187 for (
auto Flag : CLI.OutFlags)
3189 Flag.isSwiftSelf() ||
Flag.isSwiftAsync() ||
Flag.isSwiftError())
3194 OutVTs.
reserve(CLI.OutVals.size());
3196 for (
auto *Val : CLI.OutVals) {
3198 if (!isTypeLegal(Val->getType(), VT) &&
3199 !(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16))
3210 if (Callee && !computeCallAddress(Callee,
Addr))
3216 if (Subtarget->isTargetWindows() &&
Addr.getGlobalValue() &&
3217 Addr.getGlobalValue()->hasExternalWeakLinkage())
3222 if (!processCallArgs(CLI, OutVTs, NumBytes))
3226 if (
RegInfo->isAnyArgRegReserved(*MF))
3227 RegInfo->emitReservedArgRegCallError(*MF);
3231 if (Subtarget->useSmallAddressing()) {
3234 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II);
3237 else if (
Addr.getGlobalValue())
3239 else if (
Addr.getReg()) {
3245 unsigned CallReg = 0;
3247 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
3248 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
3252 CallReg = createResultReg(&AArch64::GPR64RegClass);
3253 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3254 TII.get(AArch64::LDRXui), CallReg)
3258 }
else if (
Addr.getGlobalValue())
3259 CallReg = materializeGV(
Addr.getGlobalValue());
3260 else if (
Addr.getReg())
3261 CallReg =
Addr.getReg();
3268 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).
addReg(CallReg);
3272 for (
auto Reg : CLI.OutRegs)
3282 return finishCall(CLI, NumBytes);
3287 return Len / Alignment->value() <= 4;
3292bool AArch64FastISel::tryEmitSmallMemCpy(Address Dest, Address Src,
3295 if (!isMemCpySmall(Len, Alignment))
3298 int64_t UnscaledOffset = 0;
3304 if (!Alignment || *Alignment >= 8) {
3315 assert(Alignment &&
"Alignment is set in this branch");
3317 if (Len >= 4 && *Alignment == 4)
3319 else if (Len >= 2 && *Alignment == 2)
3326 unsigned ResultReg =
emitLoad(VT, VT, Src);
3335 UnscaledOffset +=
Size;
3338 Dest.setOffset(OrigDest.getOffset() + UnscaledOffset);
3339 Src.setOffset(OrigSrc.getOffset() + UnscaledOffset);
3350 if (!isa<ExtractValueInst>(
Cond))
3353 const auto *EV = cast<ExtractValueInst>(
Cond);
3354 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
3357 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
3361 cast<StructType>(
Callee->getReturnType())->getTypeAtIndex(0U);
3362 if (!isTypeLegal(
RetTy, RetVT))
3365 if (RetVT != MVT::i32 && RetVT != MVT::i64)
3368 const Value *
LHS = II->getArgOperand(0);
3369 const Value *
RHS = II->getArgOperand(1);
3372 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && II->isCommutative())
3380 case Intrinsic::smul_with_overflow:
3381 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3382 if (
C->getValue() == 2)
3383 IID = Intrinsic::sadd_with_overflow;
3385 case Intrinsic::umul_with_overflow:
3386 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3387 if (
C->getValue() == 2)
3388 IID = Intrinsic::uadd_with_overflow;
3396 case Intrinsic::sadd_with_overflow:
3397 case Intrinsic::ssub_with_overflow:
3400 case Intrinsic::uadd_with_overflow:
3403 case Intrinsic::usub_with_overflow:
3406 case Intrinsic::smul_with_overflow:
3407 case Intrinsic::umul_with_overflow:
3413 if (!isValueAvailable(II))
3419 for (
auto Itr = std::prev(Start); Itr !=
End; --Itr) {
3422 if (!isa<ExtractValueInst>(Itr))
3426 const auto *EVI = cast<ExtractValueInst>(Itr);
3427 if (EVI->getAggregateOperand() != II)
3435bool AArch64FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *II) {
3438 default:
return false;
3439 case Intrinsic::frameaddress: {
3445 Register SrcReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3446 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3454 unsigned Depth = cast<ConstantInt>(II->
getOperand(0))->getZExtValue();
3456 DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
3458 assert(DestReg &&
"Unexpected LDR instruction emission failure.");
3462 updateValueMap(II, SrcReg);
3465 case Intrinsic::sponentry: {
3470 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
3471 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3472 TII.get(AArch64::ADDXri), ResultReg)
3477 updateValueMap(II, ResultReg);
3480 case Intrinsic::memcpy:
3481 case Intrinsic::memmove: {
3482 const auto *MTI = cast<MemTransferInst>(II);
3484 if (MTI->isVolatile())
3490 if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) {
3493 uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue();
3495 if (MTI->getDestAlign() || MTI->getSourceAlign())
3496 Alignment = std::min(MTI->getDestAlign().valueOrOne(),
3497 MTI->getSourceAlign().valueOrOne());
3498 if (isMemCpySmall(Len, Alignment)) {
3500 if (!computeAddress(MTI->getRawDest(), Dest) ||
3501 !computeAddress(MTI->getRawSource(), Src))
3503 if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment))
3508 if (!MTI->getLength()->getType()->isIntegerTy(64))
3511 if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255)
3516 const char *IntrMemName = isa<MemCpyInst>(II) ?
"memcpy" :
"memmove";
3517 return lowerCallTo(II, IntrMemName, II->
arg_size() - 1);
3519 case Intrinsic::memset: {
3520 const MemSetInst *MSI = cast<MemSetInst>(II);
3533 return lowerCallTo(II,
"memset", II->
arg_size() - 1);
3535 case Intrinsic::sin:
3536 case Intrinsic::cos:
3537 case Intrinsic::pow: {
3539 if (!isTypeLegal(II->
getType(), RetVT))
3542 if (RetVT != MVT::f32 && RetVT != MVT::f64)
3546 { RTLIB::SIN_F32, RTLIB::SIN_F64 },
3547 { RTLIB::COS_F32, RTLIB::COS_F64 },
3548 { RTLIB::POW_F32, RTLIB::POW_F64 }
3551 bool Is64Bit = RetVT == MVT::f64;
3555 case Intrinsic::sin:
3556 LC = LibCallTable[0][Is64Bit];
3558 case Intrinsic::cos:
3559 LC = LibCallTable[1][Is64Bit];
3561 case Intrinsic::pow:
3562 LC = LibCallTable[2][Is64Bit];
3570 for (
auto &Arg : II->
args()) {
3573 Entry.Ty = Arg->getType();
3574 Args.push_back(Entry);
3577 CallLoweringInfo CLI;
3579 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC), II->
getType(),
3580 TLI.getLibcallName(LC), std::move(Args));
3581 if (!lowerCallTo(CLI))
3583 updateValueMap(II, CLI.ResultReg);
3586 case Intrinsic::fabs: {
3588 if (!isTypeLegal(II->
getType(), VT))
3596 Opc = AArch64::FABSSr;
3599 Opc = AArch64::FABSDr;
3605 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3606 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
3608 updateValueMap(II, ResultReg);
3611 case Intrinsic::trap:
3612 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3615 case Intrinsic::debugtrap:
3616 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3620 case Intrinsic::sqrt: {
3624 if (!isTypeLegal(
RetTy, VT))
3631 unsigned ResultReg = fastEmit_r(VT, VT,
ISD::FSQRT, Op0Reg);
3635 updateValueMap(II, ResultReg);
3638 case Intrinsic::sadd_with_overflow:
3639 case Intrinsic::uadd_with_overflow:
3640 case Intrinsic::ssub_with_overflow:
3641 case Intrinsic::usub_with_overflow:
3642 case Intrinsic::smul_with_overflow:
3643 case Intrinsic::umul_with_overflow: {
3646 auto *Ty = cast<StructType>(
Callee->getReturnType());
3650 if (!isTypeLegal(
RetTy, VT))
3653 if (VT != MVT::i32 && VT != MVT::i64)
3659 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && II->
isCommutative())
3667 case Intrinsic::smul_with_overflow:
3668 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3669 if (
C->getValue() == 2) {
3670 IID = Intrinsic::sadd_with_overflow;
3674 case Intrinsic::umul_with_overflow:
3675 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3676 if (
C->getValue() == 2) {
3677 IID = Intrinsic::uadd_with_overflow;
3683 unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
3687 case Intrinsic::sadd_with_overflow:
3688 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3691 case Intrinsic::uadd_with_overflow:
3692 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3695 case Intrinsic::ssub_with_overflow:
3696 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3699 case Intrinsic::usub_with_overflow:
3700 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3703 case Intrinsic::smul_with_overflow: {
3705 Register LHSReg = getRegForValue(LHS);
3709 Register RHSReg = getRegForValue(RHS);
3713 if (VT == MVT::i32) {
3714 MulReg = emitSMULL_rr(MVT::i64, LHSReg, RHSReg);
3716 fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3718 emitAddSub_rx(
false, MVT::i64, MulReg, MulSubReg,
3723 assert(VT == MVT::i64 &&
"Unexpected value type.");
3726 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3727 unsigned SMULHReg = fastEmit_rr(VT, VT,
ISD::MULHS, LHSReg, RHSReg);
3733 case Intrinsic::umul_with_overflow: {
3735 Register LHSReg = getRegForValue(LHS);
3739 Register RHSReg = getRegForValue(RHS);
3743 if (VT == MVT::i32) {
3744 MulReg = emitUMULL_rr(MVT::i64, LHSReg, RHSReg);
3746 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3747 TII.get(AArch64::ANDSXri), AArch64::XZR)
3750 MulReg = fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3752 assert(VT == MVT::i64 &&
"Unexpected value type.");
3755 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3756 unsigned UMULHReg = fastEmit_rr(VT, VT,
ISD::MULHU, LHSReg, RHSReg);
3757 emitSubs_rr(VT, AArch64::XZR, UMULHReg,
false);
3764 ResultReg1 = createResultReg(TLI.getRegClassFor(VT));
3765 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3766 TII.get(TargetOpcode::COPY), ResultReg1).
addReg(MulReg);
3772 ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
3773 AArch64::WZR, AArch64::WZR,
3774 getInvertedCondCode(
CC));
3776 assert((ResultReg1 + 1) == ResultReg2 &&
3777 "Nonconsecutive result registers.");
3778 updateValueMap(II, ResultReg1, 2);
3781 case Intrinsic::aarch64_crc32b:
3782 case Intrinsic::aarch64_crc32h:
3783 case Intrinsic::aarch64_crc32w:
3784 case Intrinsic::aarch64_crc32x:
3785 case Intrinsic::aarch64_crc32cb:
3786 case Intrinsic::aarch64_crc32ch:
3787 case Intrinsic::aarch64_crc32cw:
3788 case Intrinsic::aarch64_crc32cx: {
3789 if (!Subtarget->hasCRC())
3796 case Intrinsic::aarch64_crc32b:
3797 Opc = AArch64::CRC32Brr;
3799 case Intrinsic::aarch64_crc32h:
3800 Opc = AArch64::CRC32Hrr;
3802 case Intrinsic::aarch64_crc32w:
3803 Opc = AArch64::CRC32Wrr;
3805 case Intrinsic::aarch64_crc32x:
3806 Opc = AArch64::CRC32Xrr;
3808 case Intrinsic::aarch64_crc32cb:
3809 Opc = AArch64::CRC32CBrr;
3811 case Intrinsic::aarch64_crc32ch:
3812 Opc = AArch64::CRC32CHrr;
3814 case Intrinsic::aarch64_crc32cw:
3815 Opc = AArch64::CRC32CWrr;
3817 case Intrinsic::aarch64_crc32cx:
3818 Opc = AArch64::CRC32CXrr;
3824 if (!LHSReg || !RHSReg)
3828 fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, LHSReg, RHSReg);
3829 updateValueMap(II, ResultReg);
3838 const Function &
F = *
I->getParent()->getParent();
3840 if (!FuncInfo.CanLowerReturn)
3846 if (TLI.supportSwiftError() &&
3847 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
3850 if (TLI.supportSplitCSR(FuncInfo.MF))
3856 if (
Ret->getNumOperands() > 0) {
3863 CCState CCInfo(
CC,
F.isVarArg(), *FuncInfo.MF, ValLocs,
I->getContext());
3867 if (ValLocs.
size() != 1)
3871 const Value *RV =
Ret->getOperand(0);
3889 if (!
MRI.getRegClass(SrcReg)->contains(DestReg))
3898 !Subtarget->isLittleEndian())
3902 if (RVVT == MVT::f128)
3907 if (RVVT != DestVT) {
3908 if (RVVT != MVT::i1 && RVVT != MVT::i8 && RVVT != MVT::i16)
3911 if (!Outs[0].
Flags.isZExt() && !Outs[0].Flags.isSExt())
3914 bool IsZExt = Outs[0].Flags.isZExt();
3915 SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
3923 SrcReg = emitAnd_ri(MVT::i64, SrcReg, 0xffffffff);
3926 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3927 TII.get(TargetOpcode::COPY), DestReg).
addReg(SrcReg);
3934 TII.get(AArch64::RET_ReallyLR));
3935 for (
unsigned RetReg : RetRegs)
3940bool AArch64FastISel::selectTrunc(
const Instruction *
I) {
3941 Type *DestTy =
I->getType();
3943 Type *SrcTy =
Op->getType();
3945 EVT SrcEVT = TLI.getValueType(
DL, SrcTy,
true);
3946 EVT DestEVT = TLI.getValueType(
DL, DestTy,
true);
3955 if (SrcVT != MVT::i64 && SrcVT != MVT::i32 && SrcVT != MVT::i16 &&
3958 if (DestVT != MVT::i32 && DestVT != MVT::i16 && DestVT != MVT::i8 &&
3972 if (SrcVT == MVT::i64) {
3989 Register Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg,
3992 ResultReg = emitAnd_ri(MVT::i32, Reg32, Mask);
3993 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3995 ResultReg = createResultReg(&AArch64::GPR32RegClass);
3996 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3997 TII.get(TargetOpcode::COPY), ResultReg)
4001 updateValueMap(
I, ResultReg);
4005unsigned AArch64FastISel::emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool IsZExt) {
4006 assert((DestVT == MVT::i8 || DestVT == MVT::i16 || DestVT == MVT::i32 ||
4007 DestVT == MVT::i64) &&
4008 "Unexpected value type.");
4010 if (DestVT == MVT::i8 || DestVT == MVT::i16)
4014 unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, 1);
4015 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
4016 if (DestVT == MVT::i64) {
4019 Register Reg64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4020 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4021 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4024 .
addImm(AArch64::sub_32);
4029 if (DestVT == MVT::i64) {
4033 return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
4038unsigned AArch64FastISel::emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4046 Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR;
break;
4048 Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR;
break;
4052 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4053 return fastEmitInst_rrr(Opc, RC, Op0, Op1, ZReg);
4056unsigned AArch64FastISel::emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4057 if (RetVT != MVT::i64)
4060 return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
4061 Op0, Op1, AArch64::XZR);
4064unsigned AArch64FastISel::emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4065 if (RetVT != MVT::i64)
4068 return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
4069 Op0, Op1, AArch64::XZR);
4072unsigned AArch64FastISel::emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
4075 bool NeedTrunc =
false;
4079 case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4080 case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4081 case MVT::i32: Opc = AArch64::LSLVWr;
break;
4082 case MVT::i64: Opc = AArch64::LSLVXr;
break;
4086 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4088 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4090 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4092 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4096unsigned AArch64FastISel::emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4099 "Unexpected source/return type pair.");
4100 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4101 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4102 "Unexpected source value type.");
4103 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4104 RetVT == MVT::i64) &&
"Unexpected return value type.");
4106 bool Is64Bit = (RetVT == MVT::i64);
4107 unsigned RegSize = Is64Bit ? 64 : 32;
4111 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4115 if (RetVT == SrcVT) {
4116 Register ResultReg = createResultReg(RC);
4117 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4118 TII.get(TargetOpcode::COPY), ResultReg)
4122 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4126 if (Shift >= DstBits)
4154 unsigned ImmR =
RegSize - Shift;
4156 unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift);
4157 static const unsigned OpcTable[2][2] = {
4158 {AArch64::SBFMWri, AArch64::SBFMXri},
4159 {AArch64::UBFMWri, AArch64::UBFMXri}
4161 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4162 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4164 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4165 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4168 .
addImm(AArch64::sub_32);
4171 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4174unsigned AArch64FastISel::emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
4177 bool NeedTrunc =
false;
4181 case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4182 case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4183 case MVT::i32: Opc = AArch64::LSRVWr;
break;
4184 case MVT::i64: Opc = AArch64::LSRVXr;
break;
4188 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4190 Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Mask);
4191 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4193 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4195 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4199unsigned AArch64FastISel::emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4202 "Unexpected source/return type pair.");
4203 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4204 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4205 "Unexpected source value type.");
4206 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4207 RetVT == MVT::i64) &&
"Unexpected return value type.");
4209 bool Is64Bit = (RetVT == MVT::i64);
4210 unsigned RegSize = Is64Bit ? 64 : 32;
4214 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4218 if (RetVT == SrcVT) {
4219 Register ResultReg = createResultReg(RC);
4220 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4221 TII.get(TargetOpcode::COPY), ResultReg)
4225 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4229 if (Shift >= DstBits)
4257 if (Shift >= SrcBits && IsZExt)
4258 return materializeInt(ConstantInt::get(*Context,
APInt(
RegSize, 0)), RetVT);
4263 Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4271 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4272 unsigned ImmS = SrcBits - 1;
4273 static const unsigned OpcTable[2][2] = {
4274 {AArch64::SBFMWri, AArch64::SBFMXri},
4275 {AArch64::UBFMWri, AArch64::UBFMXri}
4277 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4278 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4280 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4281 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4284 .
addImm(AArch64::sub_32);
4287 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4290unsigned AArch64FastISel::emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
4293 bool NeedTrunc =
false;
4297 case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4298 case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4299 case MVT::i32: Opc = AArch64::ASRVWr;
break;
4300 case MVT::i64: Opc = AArch64::ASRVXr;
break;
4304 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4306 Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32,
false);
4307 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4309 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4311 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4315unsigned AArch64FastISel::emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4318 "Unexpected source/return type pair.");
4319 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4320 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4321 "Unexpected source value type.");
4322 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4323 RetVT == MVT::i64) &&
"Unexpected return value type.");
4325 bool Is64Bit = (RetVT == MVT::i64);
4326 unsigned RegSize = Is64Bit ? 64 : 32;
4330 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4334 if (RetVT == SrcVT) {
4335 Register ResultReg = createResultReg(RC);
4336 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4337 TII.get(TargetOpcode::COPY), ResultReg)
4341 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4345 if (Shift >= DstBits)
4373 if (Shift >= SrcBits && IsZExt)
4374 return materializeInt(ConstantInt::get(*Context,
APInt(
RegSize, 0)), RetVT);
4376 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4377 unsigned ImmS = SrcBits - 1;
4378 static const unsigned OpcTable[2][2] = {
4379 {AArch64::SBFMWri, AArch64::SBFMXri},
4380 {AArch64::UBFMWri, AArch64::UBFMXri}
4382 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4383 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4385 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4386 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4389 .
addImm(AArch64::sub_32);
4392 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4395unsigned AArch64FastISel::emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
4397 assert(DestVT != MVT::i1 &&
"ZeroExt/SignExt an i1?");
4403 if (((DestVT != MVT::i8) && (DestVT != MVT::i16) &&
4404 (DestVT != MVT::i32) && (DestVT != MVT::i64)) ||
4405 ((SrcVT != MVT::i1) && (SrcVT != MVT::i8) &&
4406 (SrcVT != MVT::i16) && (SrcVT != MVT::i32)))
4416 return emiti1Ext(SrcReg, DestVT, IsZExt);
4418 if (DestVT == MVT::i64)
4419 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4421 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4425 if (DestVT == MVT::i64)
4426 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4428 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4432 assert(DestVT == MVT::i64 &&
"IntExt i32 to i32?!?");
4433 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4439 if (DestVT == MVT::i8 || DestVT == MVT::i16)
4441 else if (DestVT == MVT::i64) {
4442 Register Src64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4443 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4444 TII.get(AArch64::SUBREG_TO_REG), Src64)
4447 .
addImm(AArch64::sub_32);
4452 (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4453 return fastEmitInst_rii(Opc, RC, SrcReg, 0, Imm);
4460 case AArch64::LDURBBi:
4461 case AArch64::LDURHHi:
4462 case AArch64::LDURWi:
4463 case AArch64::LDRBBui:
4464 case AArch64::LDRHHui:
4465 case AArch64::LDRWui:
4466 case AArch64::LDRBBroX:
4467 case AArch64::LDRHHroX:
4468 case AArch64::LDRWroX:
4469 case AArch64::LDRBBroW:
4470 case AArch64::LDRHHroW:
4471 case AArch64::LDRWroW:
4480 case AArch64::LDURSBWi:
4481 case AArch64::LDURSHWi:
4482 case AArch64::LDURSBXi:
4483 case AArch64::LDURSHXi:
4484 case AArch64::LDURSWi:
4485 case AArch64::LDRSBWui:
4486 case AArch64::LDRSHWui:
4487 case AArch64::LDRSBXui:
4488 case AArch64::LDRSHXui:
4489 case AArch64::LDRSWui:
4490 case AArch64::LDRSBWroX:
4491 case AArch64::LDRSHWroX:
4492 case AArch64::LDRSBXroX:
4493 case AArch64::LDRSHXroX:
4494 case AArch64::LDRSWroX:
4495 case AArch64::LDRSBWroW:
4496 case AArch64::LDRSHWroW:
4497 case AArch64::LDRSBXroW:
4498 case AArch64::LDRSHXroW:
4499 case AArch64::LDRSWroW:
4504bool AArch64FastISel::optimizeIntExtLoad(
const Instruction *
I,
MVT RetVT,
4506 const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0));
4507 if (!LI || !LI->hasOneUse())
4521 bool IsZExt = isa<ZExtInst>(
I);
4522 const auto *LoadMI =
MI;
4523 if (LoadMI->getOpcode() == TargetOpcode::COPY &&
4524 LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) {
4525 Register LoadReg =
MI->getOperand(1).getReg();
4526 LoadMI =
MRI.getUniqueVRegDef(LoadReg);
4527 assert(LoadMI &&
"Expected valid instruction");
4533 if (RetVT != MVT::i64 || SrcVT > MVT::i32) {
4534 updateValueMap(
I, Reg);
4539 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
4540 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4541 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4544 .
addImm(AArch64::sub_32);
4547 assert((
MI->getOpcode() == TargetOpcode::COPY &&
4548 MI->getOperand(1).getSubReg() == AArch64::sub_32) &&
4549 "Expected copy instruction");
4550 Reg =
MI->getOperand(1).getReg();
4552 removeDeadCode(
I, std::next(
I));
4554 updateValueMap(
I, Reg);
4558bool AArch64FastISel::selectIntExt(
const Instruction *
I) {
4559 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
4560 "Unexpected integer extend instruction.");
4563 if (!isTypeSupported(
I->getType(), RetVT))
4566 if (!isTypeSupported(
I->getOperand(0)->getType(), SrcVT))
4570 if (optimizeIntExtLoad(
I, RetVT, SrcVT))
4573 Register SrcReg = getRegForValue(
I->getOperand(0));
4578 bool IsZExt = isa<ZExtInst>(
I);
4579 if (
const auto *Arg = dyn_cast<Argument>(
I->getOperand(0))) {
4580 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) {
4581 if (RetVT == MVT::i64 && SrcVT != MVT::i64) {
4582 Register ResultReg = createResultReg(&AArch64::GPR64RegClass);
4583 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4584 TII.get(AArch64::SUBREG_TO_REG), ResultReg)
4587 .
addImm(AArch64::sub_32);
4591 updateValueMap(
I, SrcReg);
4596 unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
4600 updateValueMap(
I, ResultReg);
4604bool AArch64FastISel::selectRem(
const Instruction *
I,
unsigned ISDOpcode) {
4605 EVT DestEVT = TLI.getValueType(
DL,
I->getType(),
true);
4610 if (DestVT != MVT::i64 && DestVT != MVT::i32)
4614 bool Is64bit = (DestVT == MVT::i64);
4615 switch (ISDOpcode) {
4619 DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr;
4622 DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr;
4625 unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr;
4626 Register Src0Reg = getRegForValue(
I->getOperand(0));
4630 Register Src1Reg = getRegForValue(
I->getOperand(1));
4635 (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4636 Register QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg, Src1Reg);
4637 assert(QuotReg &&
"Unexpected DIV instruction emission failure.");
4640 Register ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg, Src1Reg, Src0Reg);
4641 updateValueMap(
I, ResultReg);
4647 if (!isTypeSupported(
I->getType(), VT,
true))
4653 const Value *Src0 =
I->getOperand(0);
4654 const Value *Src1 =
I->getOperand(1);
4655 if (
const auto *
C = dyn_cast<ConstantInt>(Src0))
4656 if (
C->getValue().isPowerOf2())
4660 if (
const auto *
C = dyn_cast<ConstantInt>(Src1))
4661 if (
C->getValue().isPowerOf2()) {
4662 uint64_t ShiftVal =
C->getValue().logBase2();
4665 if (
const auto *ZExt = dyn_cast<ZExtInst>(Src0)) {
4668 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) {
4671 Src0 = ZExt->getOperand(0);
4674 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Src0)) {
4677 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) {
4680 Src0 = SExt->getOperand(0);
4685 Register Src0Reg = getRegForValue(Src0);
4689 unsigned ResultReg =
4690 emitLSL_ri(VT, SrcVT, Src0Reg, ShiftVal, IsZExt);
4693 updateValueMap(
I, ResultReg);
4698 Register Src0Reg = getRegForValue(
I->getOperand(0));
4702 Register Src1Reg = getRegForValue(
I->getOperand(1));
4706 unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src1Reg);
4711 updateValueMap(
I, ResultReg);
4715bool AArch64FastISel::selectShift(
const Instruction *
I) {
4717 if (!isTypeSupported(
I->getType(), RetVT,
true))
4721 return selectOperator(
I,
I->getOpcode());
4723 if (
const auto *
C = dyn_cast<ConstantInt>(
I->getOperand(1))) {
4724 unsigned ResultReg = 0;
4727 bool IsZExt =
I->getOpcode() != Instruction::AShr;
4728 const Value *Op0 =
I->getOperand(0);
4729 if (
const auto *ZExt = dyn_cast<ZExtInst>(Op0)) {
4732 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) {
4735 Op0 = ZExt->getOperand(0);
4738 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Op0)) {
4741 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) {
4744 Op0 = SExt->getOperand(0);
4749 Register Op0Reg = getRegForValue(Op0);
4753 switch (
I->getOpcode()) {
4755 case Instruction::Shl:
4756 ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4758 case Instruction::AShr:
4759 ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4761 case Instruction::LShr:
4762 ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4768 updateValueMap(
I, ResultReg);
4772 Register Op0Reg = getRegForValue(
I->getOperand(0));
4776 Register Op1Reg = getRegForValue(
I->getOperand(1));
4780 unsigned ResultReg = 0;
4781 switch (
I->getOpcode()) {
4783 case Instruction::Shl:
4784 ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);
4786 case Instruction::AShr:
4787 ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);
4789 case Instruction::LShr:
4790 ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);
4797 updateValueMap(
I, ResultReg);
4801bool AArch64FastISel::selectBitCast(
const Instruction *
I) {
4804 if (!isTypeLegal(
I->getOperand(0)->getType(), SrcVT))
4806 if (!isTypeLegal(
I->getType(), RetVT))
4810 if (RetVT == MVT::f32 && SrcVT == MVT::i32)
4811 Opc = AArch64::FMOVWSr;
4812 else if (RetVT == MVT::f64 && SrcVT == MVT::i64)
4813 Opc = AArch64::FMOVXDr;
4814 else if (RetVT == MVT::i32 && SrcVT == MVT::f32)
4815 Opc = AArch64::FMOVSWr;
4816 else if (RetVT == MVT::i64 && SrcVT == MVT::f64)
4817 Opc = AArch64::FMOVDXr;
4824 case MVT::i32: RC = &AArch64::GPR32RegClass;
break;
4825 case MVT::i64: RC = &AArch64::GPR64RegClass;
break;
4826 case MVT::f32: RC = &AArch64::FPR32RegClass;
break;
4827 case MVT::f64: RC = &AArch64::FPR64RegClass;
break;
4829 Register Op0Reg = getRegForValue(
I->getOperand(0));
4833 Register ResultReg = fastEmitInst_r(Opc, RC, Op0Reg);
4837 updateValueMap(
I, ResultReg);
4841bool AArch64FastISel::selectFRem(
const Instruction *
I) {
4843 if (!isTypeLegal(
I->getType(), RetVT))
4851 LC = RTLIB::REM_F32;
4854 LC = RTLIB::REM_F64;
4859 Args.reserve(
I->getNumOperands());
4862 for (
auto &Arg :
I->operands()) {
4865 Entry.Ty = Arg->getType();
4866 Args.push_back(Entry);
4869 CallLoweringInfo CLI;
4871 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC),
I->getType(),
4872 TLI.getLibcallName(LC), std::move(Args));
4873 if (!lowerCallTo(CLI))
4875 updateValueMap(
I, CLI.ResultReg);
4879bool AArch64FastISel::selectSDiv(
const Instruction *
I) {
4881 if (!isTypeLegal(
I->getType(), VT))
4884 if (!isa<ConstantInt>(
I->getOperand(1)))
4887 const APInt &
C = cast<ConstantInt>(
I->getOperand(1))->getValue();
4888 if ((VT != MVT::i32 && VT != MVT::i64) || !
C ||
4889 !(
C.isPowerOf2() ||
C.isNegatedPowerOf2()))
4892 unsigned Lg2 =
C.countr_zero();
4893 Register Src0Reg = getRegForValue(
I->getOperand(0));
4897 if (cast<BinaryOperator>(
I)->isExact()) {
4898 unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Lg2);
4901 updateValueMap(
I, ResultReg);
4905 int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
4906 unsigned AddReg = emitAdd_ri_(VT, Src0Reg, Pow2MinusOne);
4911 if (!emitICmp_ri(VT, Src0Reg, 0))
4916 if (VT == MVT::i64) {
4917 SelectOpc = AArch64::CSELXr;
4918 RC = &AArch64::GPR64RegClass;
4920 SelectOpc = AArch64::CSELWr;
4921 RC = &AArch64::GPR32RegClass;
4923 Register SelectReg = fastEmitInst_rri(SelectOpc, RC, AddReg, Src0Reg,
4930 unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
4933 ResultReg = emitAddSub_rs(
false, VT, ZeroReg, SelectReg,
4936 ResultReg = emitASR_ri(VT, VT, SelectReg, Lg2);
4941 updateValueMap(
I, ResultReg);
4948unsigned AArch64FastISel::getRegForGEPIndex(
const Value *
Idx) {
4955 MVT PtrVT = TLI.getPointerTy(
DL);
4957 if (IdxVT.
bitsLT(PtrVT)) {
4958 IdxN = emitIntExt(IdxVT.
getSimpleVT(), IdxN, PtrVT,
false);
4959 }
else if (IdxVT.
bitsGT(PtrVT))
4960 llvm_unreachable(
"AArch64 FastISel doesn't support types larger than i64");
4968bool AArch64FastISel::selectGetElementPtr(
const Instruction *
I) {
4969 if (Subtarget->isTargetILP32())
4972 Register N = getRegForValue(
I->getOperand(0));
4979 MVT VT = TLI.getPointerTy(
DL);
4982 const Value *
Idx = GTI.getOperand();
4983 if (
auto *StTy = GTI.getStructTypeOrNull()) {
4984 unsigned Field = cast<ConstantInt>(
Idx)->getZExtValue();
4987 TotalOffs +=
DL.getStructLayout(StTy)->getElementOffset(
Field);
4990 if (
const auto *CI = dyn_cast<ConstantInt>(
Idx)) {
4994 TotalOffs += GTI.getSequentialElementStride(
DL) *
4995 cast<ConstantInt>(CI)->getSExtValue();
4999 N = emitAdd_ri_(VT,
N, TotalOffs);
5006 uint64_t ElementSize = GTI.getSequentialElementStride(
DL);
5007 unsigned IdxN = getRegForGEPIndex(
Idx);
5011 if (ElementSize != 1) {
5015 IdxN = emitMul_rr(VT, IdxN,
C);
5025 N = emitAdd_ri_(VT,
N, TotalOffs);
5029 updateValueMap(
I,
N);
5034 assert(
TM.getOptLevel() == CodeGenOptLevel::None &&
5035 "cmpxchg survived AtomicExpand at optlevel > -O0");
5037 auto *RetPairTy = cast<StructType>(
I->getType());
5038 Type *
RetTy = RetPairTy->getTypeAtIndex(0U);
5039 assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) &&
5040 "cmpxchg has a non-i1 status result");
5043 if (!isTypeLegal(
RetTy, VT))
5047 unsigned Opc, CmpOpc;
5050 if (VT == MVT::i32) {
5051 Opc = AArch64::CMP_SWAP_32;
5052 CmpOpc = AArch64::SUBSWrs;
5053 ResRC = &AArch64::GPR32RegClass;
5054 }
else if (VT == MVT::i64) {
5055 Opc = AArch64::CMP_SWAP_64;
5056 CmpOpc = AArch64::SUBSXrs;
5057 ResRC = &AArch64::GPR64RegClass;
5065 II, getRegForValue(
I->getPointerOperand()), II.
getNumDefs());
5067 II, getRegForValue(
I->getCompareOperand()), II.
getNumDefs() + 1);
5069 II, getRegForValue(
I->getNewValOperand()), II.
getNumDefs() + 2);
5071 const Register ResultReg1 = createResultReg(ResRC);
5072 const Register ResultReg2 = createResultReg(&AArch64::GPR32RegClass);
5073 const Register ScratchReg = createResultReg(&AArch64::GPR32RegClass);
5076 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
5083 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(CmpOpc))
5084 .
addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR)
5089 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr))
5095 assert((ResultReg1 + 1) == ResultReg2 &&
"Nonconsecutive result registers.");
5096 updateValueMap(
I, ResultReg1, 2);
5100bool AArch64FastISel::fastSelectInstruction(
const Instruction *
I) {
5101 if (TLI.fallBackToDAGISel(*
I))
5103 switch (
I->getOpcode()) {
5106 case Instruction::Add:
5107 case Instruction::Sub:
5108 return selectAddSub(
I);
5109 case Instruction::Mul:
5110 return selectMul(
I);
5111 case Instruction::SDiv:
5112 return selectSDiv(
I);
5113 case Instruction::SRem:
5117 case Instruction::URem:
5121 case Instruction::Shl:
5122 case Instruction::LShr:
5123 case Instruction::AShr:
5124 return selectShift(
I);
5125 case Instruction::And:
5126 case Instruction::Or:
5127 case Instruction::Xor:
5128 return selectLogicalOp(
I);
5129 case Instruction::Br:
5130 return selectBranch(
I);
5131 case Instruction::IndirectBr:
5132 return selectIndirectBr(
I);
5133 case Instruction::BitCast:
5135 return selectBitCast(
I);
5137 case Instruction::FPToSI:
5139 return selectFPToInt(
I,
true);
5141 case Instruction::FPToUI:
5142 return selectFPToInt(
I,
false);
5143 case Instruction::ZExt:
5144 case Instruction::SExt:
5145 return selectIntExt(
I);
5146 case Instruction::Trunc:
5148 return selectTrunc(
I);
5150 case Instruction::FPExt:
5151 return selectFPExt(
I);
5152 case Instruction::FPTrunc:
5153 return selectFPTrunc(
I);
5154 case Instruction::SIToFP:
5156 return selectIntToFP(
I,
true);
5158 case Instruction::UIToFP:
5159 return selectIntToFP(
I,
false);
5160 case Instruction::Load:
5161 return selectLoad(
I);
5162 case Instruction::Store:
5163 return selectStore(
I);
5164 case Instruction::FCmp:
5165 case Instruction::ICmp:
5166 return selectCmp(
I);
5167 case Instruction::Select:
5168 return selectSelect(
I);
5169 case Instruction::Ret:
5170 return selectRet(
I);
5171 case Instruction::FRem:
5172 return selectFRem(
I);
5173 case Instruction::GetElementPtr:
5174 return selectGetElementPtr(
I);
5175 case Instruction::AtomicCmpXchg:
5176 return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(
I));
5180 return selectOperator(
I,
I->getOpcode());
5191 return new AArch64FastISel(FuncInfo, LibInfo);
unsigned const MachineRegisterInfo * MRI
static bool isIntExtFree(const Instruction *I)
Check if the sign-/zero-extend will be a noop.
static bool isSExtLoad(const MachineInstr *LI)
static AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred)
static bool isMulPowOf2(const Value *I)
Check if the multiply is by a power-of-2 constant.
static unsigned getImplicitScaleFactor(MVT VT)
Determine the implicit scale factor that is applied by a memory operation for a given value type.
static bool isZExtLoad(const MachineInstr *LI)
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV),...
static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator Pos, const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2, int Offset, bool IsPostDec)
Emit a load-pair instruction for frame-destroy.
static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator Pos, const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2, int Offset, bool IsPreDec)
Emit a store-pair instruction for frame-setup.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI Pre allocate WWM Registers
This file defines the SmallVector class.
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static const unsigned FramePtr
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool branchTargetEnforcement() const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
an instruction to allocate memory on the stack
PointerType * getType() const
Overload to return most specific pointer type.
This class represents an incoming formal argument to a Function.
An instruction that atomically checks whether a specified value is in a memory location,...
InstListType::const_iterator const_iterator
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
unsigned getValNo() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
A constant value that is initialized with an expression using other constant values.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the sign bit is set.
bool isZero() const
Return true if the value is positive or negative zero.
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
constexpr bool isVector() const
One or more elements.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
bool selectGetElementPtr(const User *I)
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic.
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering.
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Register getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
bool selectBitCast(const User *I)
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
PointerType * getType() const
Global values are always pointers.
Indirect Branch Instruction.
iterator_range< succ_op_iterator > successors()
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
bool isCommutative() const
Return true if swapping the first two arguments to the intrinsic produces the same result.
This is an important class for using LLVM in a threaded context.
Context object for machine code objects.
Describe properties that are true of each instruction in the target description file.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
bool is64BitVector() const
Return true if this is a 64-bit vector type.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setFrameAddressIsTaken(bool T)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
Value * getLength() const
unsigned getDestAddressSpace() const
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
Wrapper class representing virtual and physical registers.
Return a value (possibly void), from a function.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasStreamingCompatibleInterface() const
bool hasStreamingInterfaceOrBody() const
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
TypeSize getElementOffset(unsigned Idx) const
Class to represent struct types.
Provides information about what library functions are available for the current target.
Target - Wrapper for Target specific information.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isStructTy() const
True if this is an instance of StructType.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Swift
Calling convention for Swift.
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ C
The default llvm calling convention, compatible with C.
@ ADD
Simple integer binary arithmetic operators.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool CC_AArch64_Win64PCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
gep_type_iterator gep_type_end(const User *GEP)
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
AtomicOrdering
Atomic ordering for LLVM's memory model.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned getKillRegState(bool B)
bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
gep_type_iterator gep_type_begin(const User *GEP)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.