50#define GET_GICOMBINER_DEPS
51#include "AArch64GenPostLegalizeGILowering.inc"
52#undef GET_GICOMBINER_DEPS
54#define DEBUG_TYPE "aarch64-postlegalizer-lowering"
57using namespace MIPatternMatch;
58using namespace AArch64GISelUtils;
62#define GET_GICOMBINER_TYPES
63#include "AArch64GenPostLegalizeGILowering.inc"
64#undef GET_GICOMBINER_TYPES
69struct ShuffleVectorPseudo {
73 ShuffleVectorPseudo(
unsigned Opc,
Register Dst,
74 std::initializer_list<SrcOp> SrcOps)
75 : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
76 ShuffleVectorPseudo() =
default;
84 "Only possible block sizes for REV are: 16, 32, 64");
85 assert(EltSize != 64 &&
"EltSize cannot be 64 for REV mask.");
87 unsigned BlockElts =
M[0] + 1;
96 for (
unsigned i = 0; i < NumElts; ++i) {
100 if (
static_cast<unsigned>(M[i]) !=
101 (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
111 if (NumElts % 2 != 0)
113 WhichResult = (
M[0] == 0 ? 0 : 1);
114 for (
unsigned i = 0; i < NumElts; i += 2) {
115 if ((M[i] >= 0 &&
static_cast<unsigned>(M[i]) != i + WhichResult) ||
117 static_cast<unsigned>(
M[i + 1]) != i + NumElts + WhichResult))
125std::optional<std::pair<bool, uint64_t>> getExtMask(
ArrayRef<int> M,
128 auto FirstRealElt =
find_if(M, [](
int Elt) {
return Elt >= 0; });
129 if (FirstRealElt ==
M.end())
134 APInt ExpectedElt =
APInt(MaskBits, *FirstRealElt + 1);
140 [&ExpectedElt](
int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
150 bool ReverseExt =
false;
162 return std::make_pair(ReverseExt, Imm);
168 WhichResult = (
M[0] == 0 ? 0 : 1);
169 for (
unsigned i = 0; i != NumElts; ++i) {
173 if (
static_cast<unsigned>(M[i]) != 2 * i + WhichResult)
181bool isZipMask(
ArrayRef<int> M,
unsigned NumElts,
unsigned &WhichResult) {
182 if (NumElts % 2 != 0)
186 WhichResult = (
M[0] == 0 ? 0 : 1);
187 unsigned Idx = WhichResult * NumElts / 2;
188 for (
unsigned i = 0; i != NumElts; i += 2) {
189 if ((M[i] >= 0 &&
static_cast<unsigned>(M[i]) !=
Idx) ||
190 (
M[i + 1] >= 0 &&
static_cast<unsigned>(
M[i + 1]) !=
Idx + NumElts))
206 int NumInputElements) {
207 if (
M.size() !=
static_cast<size_t>(NumInputElements))
209 int NumLHSMatch = 0, NumRHSMatch = 0;
210 int LastLHSMismatch = -1, LastRHSMismatch = -1;
211 for (
int Idx = 0;
Idx < NumInputElements; ++
Idx) {
217 M[
Idx] ==
Idx ? ++NumLHSMatch : LastLHSMismatch =
Idx;
218 M[
Idx] ==
Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch =
Idx;
220 const int NumNeededToMatch = NumInputElements - 1;
221 if (NumLHSMatch == NumNeededToMatch)
222 return std::make_pair(
true, LastLHSMismatch);
223 if (NumRHSMatch == NumNeededToMatch)
224 return std::make_pair(
false, LastRHSMismatch);
231 ShuffleVectorPseudo &MatchInfo) {
232 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
236 LLT Ty =
MRI.getType(Dst);
246 for (
unsigned LaneSize : {64U, 32U, 16U}) {
247 if (
isREVMask(ShuffleMask, EltSize, NumElts, LaneSize)) {
250 Opcode = AArch64::G_REV64;
251 else if (LaneSize == 32U)
252 Opcode = AArch64::G_REV32;
254 Opcode = AArch64::G_REV16;
256 MatchInfo = ShuffleVectorPseudo(Opcode, Dst, {Src});
267 ShuffleVectorPseudo &MatchInfo) {
268 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
269 unsigned WhichResult;
272 unsigned NumElts =
MRI.getType(Dst).getNumElements();
273 if (!
isTRNMask(ShuffleMask, NumElts, WhichResult))
275 unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
278 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
288 ShuffleVectorPseudo &MatchInfo) {
289 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
290 unsigned WhichResult;
293 unsigned NumElts =
MRI.getType(Dst).getNumElements();
294 if (!
isUZPMask(ShuffleMask, NumElts, WhichResult))
296 unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
299 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
304 ShuffleVectorPseudo &MatchInfo) {
305 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
306 unsigned WhichResult;
309 unsigned NumElts =
MRI.getType(Dst).getNumElements();
310 if (!isZipMask(ShuffleMask, NumElts, WhichResult))
312 unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
315 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
322 ShuffleVectorPseudo &MatchInfo) {
341 auto *InsMI =
getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,
342 MI.getOperand(1).getReg(),
MRI);
346 if (!
getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
354 MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP,
MI.getOperand(0).getReg(),
355 {InsMI->getOperand(2).getReg()});
362 ShuffleVectorPseudo &MatchInfo) {
363 assert(Lane >= 0 &&
"Expected positive lane?");
366 auto *BuildVecMI =
getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
367 MI.getOperand(1).getReg(),
MRI);
370 Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
372 ShuffleVectorPseudo(AArch64::G_DUP,
MI.getOperand(0).getReg(), {Reg});
377 ShuffleVectorPseudo &MatchInfo) {
378 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
382 int Lane = *MaybeLane;
386 if (matchDupFromInsertVectorElt(Lane,
MI,
MRI, MatchInfo))
388 if (matchDupFromBuildVector(Lane,
MI,
MRI, MatchInfo))
405 unsigned ExpectedElt =
M[0];
406 for (
unsigned I = 1;
I < NumElts; ++
I) {
410 if (ExpectedElt == NumElts)
415 if (ExpectedElt !=
static_cast<unsigned>(M[
I]))
423 ShuffleVectorPseudo &MatchInfo) {
424 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
426 LLT DstTy =
MRI.getType(Dst);
429 auto Mask =
MI.getOperand(3).getShuffleMask();
432 uint64_t ExtFactor =
MRI.getType(V1).getScalarSizeInBits() / 8;
435 if (!getOpcodeDef<GImplicitDef>(V2,
MRI) ||
436 !isSingletonExtMask(Mask, DstTy))
440 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1,
Imm});
444 std::tie(ReverseExt, Imm) = *ExtInfo;
448 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1,
V2,
Imm});
455 ShuffleVectorPseudo &MatchInfo) {
457 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);
458 MI.eraseFromParent();
466 if (MatchInfo.SrcOps[2].getImm() == 0)
467 MIRBuilder.buildCopy(MatchInfo.Dst, MatchInfo.SrcOps[0]);
471 MIRBuilder.buildConstant(
LLT::scalar(32), MatchInfo.SrcOps[2].getImm());
472 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},
473 {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
475 MI.eraseFromParent();
490 std::tuple<Register, int, Register, int> &MatchInfo) {
491 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
494 int NumElts =
MRI.getType(Dst).getNumElements();
495 auto DstIsLeftAndDstLane =
isINSMask(ShuffleMask, NumElts);
496 if (!DstIsLeftAndDstLane)
500 std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;
506 int SrcLane = ShuffleMask[DstLane];
507 if (SrcLane >= NumElts) {
512 MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);
518 std::tuple<Register, int, Register, int> &MatchInfo) {
521 auto ScalarTy =
MRI.getType(Dst).getElementType();
523 int DstLane, SrcLane;
524 std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;
529 MI.eraseFromParent();
537 assert(Ty.
isVector() &&
"vector shift count is not a vector type");
544 return Cnt >= 1 && Cnt <= ElementBits;
550 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
551 MI.getOpcode() == TargetOpcode::G_LSHR);
552 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
560 unsigned Opc =
MI.getOpcode();
561 assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
563 Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
565 auto ImmDef = MIB.buildConstant(
LLT::scalar(32), Imm);
566 MIB.buildInstr(NewOpc, {
MI.getOperand(0)}, {
MI.getOperand(1), ImmDef});
567 MI.eraseFromParent();
577std::optional<std::pair<uint64_t, CmpInst::Predicate>>
580 const auto &Ty =
MRI.getType(RHS);
584 assert((
Size == 32 ||
Size == 64) &&
"Expected 32 or 64 bit compare only?");
591 uint64_t C = ValAndVReg->Value.getZExtValue();
609 (
Size == 32 &&
static_cast<int32_t
>(
C) == INT32_MIN))
635 if ((
Size == 32 &&
static_cast<int32_t
>(
C) == INT32_MAX) ||
649 if ((
Size == 32 &&
static_cast<uint32_t>(
C) == UINT32_MAX) ||
674bool matchAdjustICmpImmAndPred(
676 std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
677 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
680 if (
auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred,
MRI)) {
681 MatchInfo = *MaybeNewImmAndPred;
687void applyAdjustICmpImmAndPred(
688 MachineInstr &
MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
696 RHS.setReg(Cst->getOperand(0).getReg());
697 MI.getOperand(1).setPredicate(MatchInfo.second);
702 std::pair<unsigned, int> &MatchInfo) {
703 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
705 const LLT SrcTy =
MRI.getType(Src1Reg);
706 const LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
725 if (ScalarSize == 64)
726 Opc = AArch64::G_DUPLANE64;
727 else if (ScalarSize == 32)
728 Opc = AArch64::G_DUPLANE32;
731 if (ScalarSize == 32)
732 Opc = AArch64::G_DUPLANE32;
733 else if (ScalarSize == 16)
734 Opc = AArch64::G_DUPLANE16;
738 Opc = AArch64::G_DUPLANE8;
739 else if (ScalarSize == 16)
740 Opc = AArch64::G_DUPLANE16;
744 Opc = AArch64::G_DUPLANE8;
752 MatchInfo.first = Opc;
753 MatchInfo.second = *LaneIdx;
759 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
761 const LLT SrcTy =
MRI.getType(Src1Reg);
763 B.setInstrAndDebugLoc(
MI);
764 auto Lane =
B.buildConstant(
LLT::scalar(64), MatchInfo.second);
770 auto Undef =
B.buildUndef(SrcTy);
772 {Src1Reg, Undef.getReg(0)})
775 B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});
776 MI.eraseFromParent();
780 auto &Unmerge = cast<GUnmerge>(
MI);
781 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
782 const LLT SrcTy =
MRI.getType(Src1Reg);
791 auto &Unmerge = cast<GUnmerge>(
MI);
792 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
793 const LLT SrcTy =
MRI.getType(Src1Reg);
795 "Expected a fixed length vector");
798 B.buildExtractVectorElementConstant(Unmerge.getReg(
I), Src1Reg,
I);
799 MI.eraseFromParent();
803 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
812 int64_t Cst =
Splat->getCst();
813 return (Cst != 0 && Cst != -1);
818 B.setInstrAndDebugLoc(
MI);
819 B.buildInstr(AArch64::G_DUP, {
MI.getOperand(0).
getReg()},
821 MI.eraseFromParent();
828 if (!
MRI.hasOneNonDBGUse(CmpOp))
833 if (
MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
835 if (
MI.getOpcode() != TargetOpcode::G_AND)
842 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
846 if (IsSupportedExtend(*Def))
849 unsigned Opc =
Def->getOpcode();
850 if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&
851 Opc != TargetOpcode::G_LSHR)
858 uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
865 if (IsSupportedExtend(*ShiftLHS))
866 return (ShiftAmt <= 4) ? 2 : 1;
868 LLT Ty =
MRI.getType(
Def->getOperand(0).getReg());
872 if ((ShiftSize == 32 && ShiftAmt <= 31) ||
873 (ShiftSize == 64 && ShiftAmt <= 63))
881 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
907 Register TheLHS = GetRegForProfit(LHS);
908 Register TheRHS = GetRegForProfit(RHS);
922 MI.getOperand(2).setReg(RHS);
923 MI.getOperand(3).setReg(LHS);
934 LLT DstTy =
MRI.getType(LHS);
936 assert(DstTy ==
MRI.getType(RHS) &&
"Src and Dst types must match!");
988 assert(
MI.getOpcode() == TargetOpcode::G_FCMP);
992 LLT DstTy =
MRI.getType(Dst);
996 unsigned EltSize =
MRI.getType(LHS).getScalarSizeInBits();
997 if (EltSize == 16 && !
ST.hasFullFP16())
999 if (EltSize != 16 && EltSize != 32 && EltSize != 64)
1008 assert(
MI.getOpcode() == TargetOpcode::G_FCMP);
1011 const auto &CmpMI = cast<GFCmp>(
MI);
1018 LLT DstTy =
MRI.getType(Dst);
1025 bool Invert =
false;
1027 if ((Pred == CmpInst::Predicate::FCMP_ORD ||
1028 Pred == CmpInst::Predicate::FCMP_UNO) &&
1045 ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
1047 auto Cmp = getVectorFCMP(
CC, LHS, RHS, IsZero, NoNans,
MRI);
1052 auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans,
MRI);
1053 auto Cmp2Dst = Cmp2(MIB);
1054 auto Cmp1Dst =
Cmp(MIB);
1059 MRI.replaceRegWith(Dst, CmpRes);
1060 MI.eraseFromParent();
1065 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
1067 if (
MRI.getType(DstReg).isVector())
1073 return MRI.getType(SrcReg).getSizeInBits() <= 64;
1079 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
1081 MI.getOperand(0).setReg(SrcReg);
1089 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1091 LLT DstTy =
MRI.getType(DstReg);
1097 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1098 B.setInstrAndDebugLoc(
MI);
1100 Helper.lower(
MI, 0,
LLT());
1107 auto &Unmerge = cast<GUnmerge>(
MI);
1108 if (Unmerge.getNumDefs() != 2)
1110 if (!
MRI.use_nodbg_empty(Unmerge.getReg(1)))
1113 LLT DstTy =
MRI.getType(Unmerge.getReg(0));
1125 if (!LowestVal || LowestVal->Value.getZExtValue() != DstTy.
getSizeInBytes())
1128 if (!getOpcodeDef<GImplicitDef>(ExtSrc2,
MRI))
1131 MatchInfo = ExtSrc1;
1141 MI.getOperand(0).setReg(
MI.getOperand(1).getReg());
1142 MI.getOperand(1).setReg(Dst1);
1143 MI.getOperand(2).setReg(SrcReg);
1154 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1160 unsigned I1Opc =
I1->getOpcode();
1162 if (((I1Opc == TargetOpcode::G_ZEXT && I2Opc == TargetOpcode::G_ZEXT) ||
1163 (I1Opc == TargetOpcode::G_SEXT && I2Opc == TargetOpcode::G_SEXT)) &&
1164 (
MRI.getType(
I1->getOperand(0).getReg()).getScalarSizeInBits() ==
1165 MRI.getType(
I1->getOperand(1).getReg()).getScalarSizeInBits() * 2) &&
1180 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
1181 "Expected a G_MUL instruction");
1184 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1189 unsigned I1Opc =
I1->getOpcode();
1191 if (((I1Opc == TargetOpcode::G_ZEXT && I2Opc == TargetOpcode::G_ZEXT) ||
1192 (I1Opc == TargetOpcode::G_SEXT && I2Opc == TargetOpcode::G_SEXT)) &&
1193 (
MRI.getType(
I1->getOperand(0).getReg()).getScalarSizeInBits() ==
1194 MRI.getType(
I1->getOperand(1).getReg()).getScalarSizeInBits() * 2) &&
1198 B.setInstrAndDebugLoc(
MI);
1199 B.buildInstr(
I1->getOpcode() == TargetOpcode::G_ZEXT ? AArch64::G_UMULL
1201 {MI.getOperand(0).getReg()},
1202 {I1->getOperand(1).getReg(), I2->getOperand(1).getReg()});
1203 MI.eraseFromParent();
1208 B.setInstrAndDebugLoc(
MI);
1209 Helper.fewerElementsVector(
1216class AArch64PostLegalizerLoweringImpl :
public Combiner {
1220 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig;
1224 AArch64PostLegalizerLoweringImpl(
1227 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1230 static const char *
getName() {
return "AArch6400PreLegalizerCombiner"; }
1235#define GET_GICOMBINER_CLASS_MEMBERS
1236#include "AArch64GenPostLegalizeGILowering.inc"
1237#undef GET_GICOMBINER_CLASS_MEMBERS
1240#define GET_GICOMBINER_IMPL
1241#include "AArch64GenPostLegalizeGILowering.inc"
1242#undef GET_GICOMBINER_IMPL
1244AArch64PostLegalizerLoweringImpl::AArch64PostLegalizerLoweringImpl(
1247 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1249 :
Combiner(MF, CInfo, TPC, nullptr, CSEInfo),
1250 Helper(Observer,
B,
true), RuleConfig(RuleConfig),
1253#include
"AArch64GenPostLegalizeGILowering.inc"
1262 AArch64PostLegalizerLowering();
1265 return "AArch64PostLegalizerLowering";
1272 AArch64PostLegalizerLoweringImplRuleConfig RuleConfig;
1276void AArch64PostLegalizerLowering::getAnalysisUsage(
AnalysisUsage &AU)
const {
1283AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()
1287 if (!RuleConfig.parseCommandLineOption())
1291bool AArch64PostLegalizerLowering::runOnMachineFunction(
MachineFunction &MF) {
1293 MachineFunctionProperties::Property::FailedISel))
1296 MachineFunctionProperties::Property::Legalized) &&
1297 "Expected a legalized function?");
1298 auto *TPC = &getAnalysis<TargetPassConfig>();
1304 F.hasOptSize(),
F.hasMinSize());
1305 AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, TPC,
nullptr,
1307 return Impl.combineMachineInstrs();
1310char AArch64PostLegalizerLowering::ID = 0;
1312 "Lower AArch64 MachineInstrs after legalization",
false,
1321 return new AArch64PostLegalizerLowering();
unsigned const MachineRegisterInfo * MRI
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isLegalArithImmed(uint64_t C)
static bool isREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
static bool isUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isINSMask(ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static unsigned getCmpOperandFoldingProfit(SDValue Op)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static bool isCMN(SDValue Op, ISD::CondCode CC)
This file declares the targeting of the Machinelegalizer class for AArch64.
#define GET_GICOMBINER_CONSTRUCTOR_INITS
Lower AArch64 MachineInstrs after legalization
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This contains common combine transformations that may be used in a combine pass,or by the target else...
Option class for Targets to specify which operations are combined how and when.
This contains the base class for all Combiners generated by TableGen.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This contains common code to allow clients to notify changes to machine instr.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const int BlockSize
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned logBase2() const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
virtual bool tryCombineAll(MachineInstr &I) const =0
FunctionPass class - This class is used to implement most global optimizations.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr LLT multiplyElements(int Factor) const
Produce a vector type that is Factor times bigger, preserving the element type.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr bool isScalable() const
Returns true if the LLT is a scalable vector.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Helper class to build MachineInstr.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Target-Independent Code Generator Pass Configuration Options.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::optional< RegOrConstant > getAArch64VectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
void changeVectorFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
Find the AArch64 condition codes necessary to represent P for a vector floating point comparison.
std::optional< int64_t > getAArch64VectorSplatScalar(const MachineInstr &MI, const MachineRegisterInfo &MRI)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
operand_type_match m_Reg()
SpecificConstantMatch m_ZeroInt()
{ Convenience matchers for specific integer values.
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
void initializeAArch64PostLegalizerLoweringPass(PassRegistry &)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
FunctionPass * createAArch64PostLegalizerLowering()
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.