43#define GET_GICOMBINER_DEPS
44#include "AArch64GenPostLegalizeGICombiner.inc"
45#undef GET_GICOMBINER_DEPS
47#define DEBUG_TYPE "aarch64-postlegalizer-combiner"
50using namespace MIPatternMatch;
54#define GET_GICOMBINER_TYPES
55#include "AArch64GenPostLegalizeGICombiner.inc"
56#undef GET_GICOMBINER_TYPES
66bool matchExtractVecEltPairwiseAdd(
68 std::tuple<unsigned, LLT, Register> &MatchInfo) {
71 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
74 if (!Cst || Cst->Value != 0)
85 if (DstSize != 16 && DstSize != 32 && DstSize != 64)
88 Register Src1Op1 = FAddMI->getOperand(1).getReg();
89 Register Src1Op2 = FAddMI->getOperand(2).getReg();
101 std::get<0>(MatchInfo) = TargetOpcode::G_FADD;
102 std::get<1>(MatchInfo) = DstTy;
103 std::get<2>(MatchInfo) =
Other->getOperand(0).getReg();
109void applyExtractVecEltPairwiseAdd(
111 std::tuple<unsigned, LLT, Register> &MatchInfo) {
112 unsigned Opc = std::get<0>(MatchInfo);
113 assert(Opc == TargetOpcode::G_FADD &&
"Unexpected opcode!");
115 LLT Ty = std::get<1>(MatchInfo);
116 Register Src = std::get<2>(MatchInfo);
118 B.setInstrAndDebugLoc(
MI);
119 auto Elt0 =
B.buildExtractVectorElement(Ty, Src,
B.buildConstant(s64, 0));
120 auto Elt1 =
B.buildExtractVectorElement(Ty, Src,
B.buildConstant(s64, 1));
121 B.buildInstr(Opc, {
MI.getOperand(0).
getReg()}, {Elt0, Elt1});
122 MI.eraseFromParent();
127 unsigned Opc =
MRI.getVRegDef(R)->getOpcode();
128 return Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG;
133 return MRI.getVRegDef(R)->getOpcode() == TargetOpcode::G_ZEXT;
136bool matchAArch64MulConstCombine(
139 assert(
MI.getOpcode() == TargetOpcode::G_MUL);
143 const LLT Ty =
MRI.getType(LHS);
164 unsigned TrailingZeroes = ConstValue.
countr_zero();
165 if (TrailingZeroes) {
168 if (
MRI.hasOneNonDBGUse(LHS) &&
173 if (
MRI.hasOneNonDBGUse(Dst)) {
175 unsigned UseOpc =
UseMI.getOpcode();
176 if (UseOpc == TargetOpcode::G_ADD || UseOpc == TargetOpcode::G_PTR_ADD ||
177 UseOpc == TargetOpcode::G_SUB)
183 APInt ShiftedConstValue = ConstValue.
ashr(TrailingZeroes);
185 unsigned ShiftAmt, AddSubOpc;
187 bool ShiftValUseIsLHS =
true;
189 bool NegateResult =
false;
195 APInt SCVMinus1 = ShiftedConstValue - 1;
196 APInt CVPlus1 = ConstValue + 1;
199 AddSubOpc = TargetOpcode::G_ADD;
202 AddSubOpc = TargetOpcode::G_SUB;
208 APInt CVNegPlus1 = -ConstValue + 1;
209 APInt CVNegMinus1 = -ConstValue - 1;
212 AddSubOpc = TargetOpcode::G_SUB;
213 ShiftValUseIsLHS =
false;
216 AddSubOpc = TargetOpcode::G_ADD;
222 if (NegateResult && TrailingZeroes)
226 auto Shift =
B.buildConstant(
LLT::scalar(64), ShiftAmt);
227 auto ShiftedVal =
B.buildShl(Ty, LHS, Shift);
229 Register AddSubLHS = ShiftValUseIsLHS ? ShiftedVal.getReg(0) :
LHS;
230 Register AddSubRHS = ShiftValUseIsLHS ?
LHS : ShiftedVal.getReg(0);
231 auto Res =
B.buildInstr(AddSubOpc, {Ty}, {AddSubLHS, AddSubRHS});
232 assert(!(NegateResult && TrailingZeroes) &&
233 "NegateResult and TrailingZeroes cannot both be true for now.");
236 B.buildSub(DstReg,
B.buildConstant(Ty, 0), Res);
240 if (TrailingZeroes) {
241 B.buildShl(DstReg, Res,
B.buildConstant(
LLT::scalar(64), TrailingZeroes));
244 B.buildCopy(DstReg, Res.getReg(0));
249void applyAArch64MulConstCombine(
252 B.setInstrAndDebugLoc(
MI);
253 ApplyFn(
B,
MI.getOperand(0).getReg());
254 MI.eraseFromParent();
260 auto &
Merge = cast<GMerge>(
MI);
273 MI.setDesc(
B.getTII().get(TargetOpcode::G_ZEXT));
288 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT);
291 return MRI.getType(Dst).isScalar() &&
301 MI.setDesc(
B.getTII().get(TargetOpcode::G_ZEXT));
309 if (!
Store.isSimple())
316 if (!
MRI.hasOneNonDBGUse(
Store.getValueReg()))
320 return MaybeCst && MaybeCst->isZero();
326 B.setInstrAndDebugLoc(
MI);
329 "Expected a vector store value");
332 auto Zero =
B.buildConstant(NewTy, 0);
333 auto HighPtr =
B.buildPtrAdd(
MRI.getType(PtrReg), PtrReg,
335 auto &MF = *
MI.getMF();
336 auto *LowMMO = MF.getMachineMemOperand(&
Store.getMMO(), 0, NewTy);
337 auto *HighMMO = MF.getMachineMemOperand(&
Store.getMMO(), 8, NewTy);
338 B.buildStore(Zero, PtrReg, *LowMMO);
339 B.buildStore(Zero, HighPtr, *HighMMO);
340 Store.eraseFromParent();
344 std::tuple<Register, Register, Register> &MatchInfo) {
345 const LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
355 auto *BV1 = getOpcodeDef<GBuildVector>(BVO1,
MRI);
356 auto *BV2 = getOpcodeDef<GBuildVector>(BVO2,
MRI);
365 if (!ValAndVReg1 || !ValAndVReg2 ||
366 ValAndVReg1->Value != ~ValAndVReg2->Value)
370 MatchInfo = {AO1, AO2, BVO1};
376 std::tuple<Register, Register, Register> &MatchInfo) {
377 B.setInstrAndDebugLoc(
MI);
379 AArch64::G_BSP, {
MI.getOperand(0).
getReg()},
380 {std::get<2>(MatchInfo), std::get<0>(MatchInfo), std::get<1>(MatchInfo)});
381 MI.eraseFromParent();
384class AArch64PostLegalizerCombinerImpl :
public Combiner {
388 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig;
392 AArch64PostLegalizerCombinerImpl(
395 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
399 static const char *
getName() {
return "AArch64PostLegalizerCombiner"; }
404#define GET_GICOMBINER_CLASS_MEMBERS
405#include "AArch64GenPostLegalizeGICombiner.inc"
406#undef GET_GICOMBINER_CLASS_MEMBERS
409#define GET_GICOMBINER_IMPL
410#include "AArch64GenPostLegalizeGICombiner.inc"
411#undef GET_GICOMBINER_IMPL
413AArch64PostLegalizerCombinerImpl::AArch64PostLegalizerCombinerImpl(
416 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
419 :
Combiner(MF, CInfo, TPC, &KB, CSEInfo),
420 Helper(Observer,
B,
false, &KB, MDT, LI),
421 RuleConfig(RuleConfig), STI(STI),
423#include
"AArch64GenPostLegalizeGICombiner.inc"
432 AArch64PostLegalizerCombiner(
bool IsOptNone =
false);
435 return "AArch64PostLegalizerCombiner";
443 AArch64PostLegalizerCombinerImplRuleConfig RuleConfig;
463void AArch64PostLegalizerCombiner::getAnalysisUsage(
AnalysisUsage &AU)
const {
478AArch64PostLegalizerCombiner::AArch64PostLegalizerCombiner(
bool IsOptNone)
482 if (!RuleConfig.parseCommandLineOption())
486bool AArch64PostLegalizerCombiner::runOnMachineFunction(
MachineFunction &MF) {
488 MachineFunctionProperties::Property::FailedISel))
491 MachineFunctionProperties::Property::Legalized) &&
492 "Expected a legalized function?");
493 auto *TPC = &getAnalysis<TargetPassConfig>();
499 const auto *LI =
ST.getLegalizerInfo();
501 GISelKnownBits *KB = &getAnalysis<GISelKnownBitsAnalysis>().get(MF);
503 IsOptNone ? nullptr : &getAnalysis<MachineDominatorTree>();
505 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
506 auto *CSEInfo = &
Wrapper.get(TPC->getCSEConfig());
509 nullptr, EnableOpt,
F.hasOptSize(),
511 AArch64PostLegalizerCombinerImpl Impl(MF, CInfo, TPC, *KB, CSEInfo,
512 RuleConfig, ST, MDT, LI);
513 bool Changed = Impl.combineMachineInstrs();
516 MIB.setCSEInfo(CSEInfo);
517 Changed |= optimizeConsecutiveMemOpAddressing(MF, MIB);
521bool AArch64PostLegalizerCombiner::tryOptimizeConsecStores(
523 if (Stores.
size() <= 2)
527 int64_t BaseOffset = Stores[0].Offset;
528 unsigned NumPairsExpected = Stores.
size() / 2;
529 unsigned TotalInstsExpected = NumPairsExpected + (Stores.
size() % 2);
533 if (!TLI.isLegalAddImmediate(BaseOffset))
534 TotalInstsExpected++;
535 int SavingsExpected = Stores.
size() - TotalInstsExpected;
536 if (SavingsExpected <= 0)
543 Register NewBase = Stores[0].Ptr->getReg(0);
544 for (
auto &SInfo : Stores) {
548 auto NewPtr = MIB.
buildPtrAdd(
MRI.getType(SInfo.St->getPointerReg()),
552 SInfo.St->getOperand(1).setReg(NewPtr.getReg(0));
557 <<
" stores into a base pointer and offsets.\n");
564 cl::desc(
"Enable consecutive memop optimization "
565 "in AArch64PostLegalizerCombiner"));
567bool AArch64PostLegalizerCombiner::optimizeConsecutiveMemOpAddressing(
590 bool Changed =
false;
609 if (
Last.Ptr->getBaseReg() != New.Ptr->getBaseReg() ||
610 (
Last.Offset +
static_cast<int64_t
>(
Last.StoredType.getSizeInBytes()) !=
612 Last.StoredType != New.StoredType)
618 return New.St->getValueReg() == LoadVal;
625 int64_t MaxLegalOffset;
626 switch (
New.StoredType.getSizeInBits()) {
628 MaxLegalOffset = 252;
631 MaxLegalOffset = 504;
634 MaxLegalOffset = 1008;
639 if (
New.Offset < MaxLegalOffset)
643 return New.Offset - Stores[0].Offset <= MaxLegalOffset;
646 auto resetState = [&]() {
648 LoadValsSinceLastStore.
clear();
651 for (
auto &
MBB : MF) {
655 for (
auto &
MI :
MBB) {
656 if (
auto *St = dyn_cast<GStore>(&
MI)) {
659 LLT StoredValTy =
MRI.getType(St->getValueReg());
661 if (ValSize < 32 || ValSize != St->getMMO().getSizeInBits())
664 Register PtrReg = St->getPointerReg();
668 GPtrAdd *PtrAdd = cast<GPtrAdd>(
MRI.getVRegDef(PtrReg));
671 if (Stores.
empty()) {
678 if (storeIsValid(
Last, New)) {
680 LoadValsSinceLastStore.
clear();
684 Changed |= tryOptimizeConsecStores(Stores, MIB);
689 }
else if (
auto *Ld = dyn_cast<GLoad>(&
MI)) {
690 LoadValsSinceLastStore.
push_back(Ld->getDstReg());
693 Changed |= tryOptimizeConsecStores(Stores, MIB);
700char AArch64PostLegalizerCombiner::ID = 0;
702 "Combine AArch64 MachineInstrs after legalization",
false,
712 return new AArch64PostLegalizerCombiner(IsOptNone);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
#define GET_GICOMBINER_CONSTRUCTOR_INITS
static cl::opt< bool > EnableConsecutiveMemOpOpt("aarch64-postlegalizer-consecutive-memops", cl::init(true), cl::Hidden, cl::desc("Enable consecutive memop optimization " "in AArch64PostLegalizerCombiner"))
Combine AArch64 MachineInstrs after legalization
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This contains common combine transformations that may be used in a combine pass,or by the target else...
Option class for Targets to specify which operations are combined how and when.
This contains the base class for all Combiners generated by TableGen.
std::optional< std::vector< StOtherPiece > > Other
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned logBase2() const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Defines a builder that does CSE of MachineInstructions using GISelCSEInfo.
MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val) override
Build and insert Res = G_CONSTANT Val.
virtual bool tryCombineAll(MachineInstr &I) const =0
FunctionPass class - This class is used to implement most global optimizations.
The actual analysis pass wrapper.
Simple wrapper that does the following.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
To use KnownBitsInfo analysis in a pass, KnownBitsInfo &Info = getAnalysis<GISelKnownBitsInfoAnalysis...
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineFunctionProperties & getProperties() const
Get the function properties.
Helper class to build MachineInstr.
GISelChangeObserver * getObserver()
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
ArrayRef< int > getShuffleMask() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Target-Independent Code Generator Pass Configuration Options.
virtual const TargetLowering * getTargetLowering() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
operand_type_match m_Pred()
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
void initializeAArch64PostLegalizerCombinerPass(PassRegistry &)
std::optional< APInt > isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a splat vector of constant integers.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
FunctionPass * createAArch64PostLegalizerCombiner(bool IsOptNone)
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...