40#define DEBUG_TYPE "aarch64-postlegalizer-combiner"
43using namespace MIPatternMatch;
55 std::tuple<unsigned, LLT, Register> &MatchInfo) {
58 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
61 if (!Cst || Cst->Value != 0)
72 if (DstSize != 16 && DstSize != 32 && DstSize != 64)
75 Register Src1Op1 = FAddMI->getOperand(1).getReg();
76 Register Src1Op2 = FAddMI->getOperand(2).getReg();
88 std::get<0>(MatchInfo) = TargetOpcode::G_FADD;
89 std::get<1>(MatchInfo) = DstTy;
90 std::get<2>(MatchInfo) =
Other->getOperand(0).getReg();
98 std::tuple<unsigned, LLT, Register> &MatchInfo) {
99 unsigned Opc = std::get<0>(MatchInfo);
100 assert(Opc == TargetOpcode::G_FADD &&
"Unexpected opcode!");
102 LLT Ty = std::get<1>(MatchInfo);
103 Register Src = std::get<2>(MatchInfo);
105 B.setInstrAndDebugLoc(
MI);
106 auto Elt0 =
B.buildExtractVectorElement(Ty, Src,
B.buildConstant(s64, 0));
107 auto Elt1 =
B.buildExtractVectorElement(Ty, Src,
B.buildConstant(s64, 1));
108 B.buildInstr(Opc, {
MI.getOperand(0).
getReg()}, {Elt0, Elt1});
109 MI.eraseFromParent();
115 unsigned Opc =
MRI.getVRegDef(R)->getOpcode();
116 return Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG;
121 return MRI.getVRegDef(R)->getOpcode() == TargetOpcode::G_ZEXT;
127 assert(
MI.getOpcode() == TargetOpcode::G_MUL);
152 unsigned TrailingZeroes = ConstValue.
countr_zero();
153 if (TrailingZeroes) {
156 if (
MRI.hasOneNonDBGUse(
LHS) &&
161 if (
MRI.hasOneNonDBGUse(Dst)) {
163 unsigned UseOpc =
UseMI.getOpcode();
164 if (UseOpc == TargetOpcode::G_ADD || UseOpc == TargetOpcode::G_PTR_ADD ||
165 UseOpc == TargetOpcode::G_SUB)
171 APInt ShiftedConstValue = ConstValue.
ashr(TrailingZeroes);
173 unsigned ShiftAmt, AddSubOpc;
175 bool ShiftValUseIsLHS =
true;
177 bool NegateResult =
false;
183 APInt SCVMinus1 = ShiftedConstValue - 1;
184 APInt CVPlus1 = ConstValue + 1;
187 AddSubOpc = TargetOpcode::G_ADD;
190 AddSubOpc = TargetOpcode::G_SUB;
196 APInt CVNegPlus1 = -ConstValue + 1;
197 APInt CVNegMinus1 = -ConstValue - 1;
200 AddSubOpc = TargetOpcode::G_SUB;
201 ShiftValUseIsLHS =
false;
204 AddSubOpc = TargetOpcode::G_ADD;
210 if (NegateResult && TrailingZeroes)
214 auto Shift =
B.buildConstant(
LLT::scalar(64), ShiftAmt);
215 auto ShiftedVal =
B.buildShl(Ty,
LHS, Shift);
217 Register AddSubLHS = ShiftValUseIsLHS ? ShiftedVal.getReg(0) :
LHS;
218 Register AddSubRHS = ShiftValUseIsLHS ?
LHS : ShiftedVal.getReg(0);
219 auto Res =
B.buildInstr(AddSubOpc, {Ty}, {AddSubLHS, AddSubRHS});
220 assert(!(NegateResult && TrailingZeroes) &&
221 "NegateResult and TrailingZeroes cannot both be true for now.");
224 B.buildSub(DstReg,
B.buildConstant(Ty, 0), Res);
228 if (TrailingZeroes) {
229 B.buildShl(DstReg, Res,
B.buildConstant(
LLT::scalar(64), TrailingZeroes));
232 B.buildCopy(DstReg, Res.getReg(0));
240 B.setInstrAndDebugLoc(
MI);
241 ApplyFn(
B,
MI.getOperand(0).getReg());
242 MI.eraseFromParent();
249 auto &
Merge = cast<GMerge>(
MI);
262 MI.setDesc(
B.getTII().get(TargetOpcode::G_ZEXT));
277 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT);
280 return MRI.getType(Dst).isScalar() &&
290 MI.setDesc(
B.getTII().get(TargetOpcode::G_ZEXT));
298 if (!Store.isSimple())
300 LLT ValTy =
MRI.getType(Store.getValueReg());
305 if (!
MRI.hasOneNonDBGUse(Store.getValueReg()))
308 *
MRI.getVRegDef(Store.getValueReg()),
MRI);
309 return MaybeCst && MaybeCst->isZero();
315 B.setInstrAndDebugLoc(
MI);
317 assert(
MRI.getType(Store.getValueReg()).isVector() &&
318 "Expected a vector store value");
320 Register PtrReg = Store.getPointerReg();
321 auto Zero =
B.buildConstant(NewTy, 0);
322 auto HighPtr =
B.buildPtrAdd(
MRI.getType(PtrReg), PtrReg,
324 auto &MF = *
MI.getMF();
325 auto *LowMMO = MF.getMachineMemOperand(&Store.getMMO(), 0, NewTy);
326 auto *HighMMO = MF.getMachineMemOperand(&Store.getMMO(), 8, NewTy);
327 B.buildStore(Zero, PtrReg, *LowMMO);
328 B.buildStore(Zero, HighPtr, *HighMMO);
329 Store.eraseFromParent();
332#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
333#include "AArch64GenPostLegalizeGICombiner.inc"
334#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
337#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
338#include "AArch64GenPostLegalizeGICombiner.inc"
339#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
341class AArch64PostLegalizerCombinerInfo :
public CombinerInfo {
346 AArch64GenPostLegalizerCombinerHelperRuleConfig GeneratedRuleCfg;
348 AArch64PostLegalizerCombinerInfo(
bool EnableOpt,
bool OptSize,
bool MinSize,
352 nullptr, EnableOpt, OptSize, MinSize),
354 if (!GeneratedRuleCfg.parseCommandLineOption())
366 MI.getParent()->getParent()->getSubtarget().getLegalizerInfo();
368 AArch64GenPostLegalizerCombinerHelper
Generated(GeneratedRuleCfg);
369 return Generated.tryCombineAll(Observer,
MI,
B, Helper);
372#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
373#include "AArch64GenPostLegalizeGICombiner.inc"
374#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
380 AArch64PostLegalizerCombiner(
bool IsOptNone =
false);
383 return "AArch64PostLegalizerCombiner";
394void AArch64PostLegalizerCombiner::getAnalysisUsage(
AnalysisUsage &AU)
const {
409AArch64PostLegalizerCombiner::AArch64PostLegalizerCombiner(
bool IsOptNone)
414bool AArch64PostLegalizerCombiner::runOnMachineFunction(
MachineFunction &MF) {
416 MachineFunctionProperties::Property::FailedISel))
419 MachineFunctionProperties::Property::Legalized) &&
420 "Expected a legalized function?");
421 auto *TPC = &getAnalysis<TargetPassConfig>();
425 GISelKnownBits *KB = &getAnalysis<GISelKnownBitsAnalysis>().get(MF);
427 IsOptNone ? nullptr : &getAnalysis<MachineDominatorTree>();
428 AArch64PostLegalizerCombinerInfo PCInfo(EnableOpt,
F.hasOptSize(),
429 F.hasMinSize(), KB, MDT);
431 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
432 auto *CSEInfo = &
Wrapper.get(TPC->getCSEConfig());
434 return C.combineMachineInstrs(MF, CSEInfo);
437char AArch64PostLegalizerCombiner::ID = 0;
439 "Combine AArch64 MachineInstrs after legalization",
false,
449 return new AArch64PostLegalizerCombiner(IsOptNone);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
bool matchExtractVecEltPairwiseAdd(MachineInstr &MI, MachineRegisterInfo &MRI, std::tuple< unsigned, LLT, Register > &MatchInfo)
This combine tries do what performExtractVectorEltCombine does in SDAG.
void applyFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer)
bool matchFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI)
Try to fold a G_MERGE_VALUES of 2 s32 sources, where the second source is a zero, into a G_ZEXT of th...
static bool isZeroExtended(Register R, MachineRegisterInfo &MRI)
static void applySplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer)
bool applyAArch64MulConstCombine(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, std::function< void(MachineIRBuilder &B, Register DstReg)> &ApplyFn)
bool applyExtractVecEltPairwiseAdd(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, std::tuple< unsigned, LLT, Register > &MatchInfo)
static bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI)
Match a 128b store of zero and split it into two 64 bit stores, for size/performance reasons.
bool matchAArch64MulConstCombine(MachineInstr &MI, MachineRegisterInfo &MRI, std::function< void(MachineIRBuilder &B, Register DstReg)> &ApplyFn)
static bool isSignExtended(Register R, MachineRegisterInfo &MRI)
Combine AArch64 MachineInstrs after legalization
static void applyMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer)
static bool matchMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI)
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Provides analysis for continuously CSEing during GISel passes.
This contains common combine transformations that may be used in a combine pass,or by the target else...
Interface for Targets to specify which operations are combined how and when.
This contains common code to drive combines.
std::optional< std::vector< StOtherPiece > > Other
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned logBase2() const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI, MachineIRBuilder &B) const =0
Attempt to combine instructions using MI as the root.
FunctionPass class - This class is used to implement most global optimizations.
The actual analysis pass wrapper.
Simple wrapper that does the following.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
To use KnownBitsInfo analysis in a pass, KnownBitsInfo &Info = getAnalysis<GISelKnownBitsInfoAnalysis...
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineFunctionProperties & getProperties() const
Get the function properties.
Helper class to build MachineInstr.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
ArrayRef< int > getShuffleMask() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
StringRef - Represent a constant reference to a string, i.e.
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Target-Independent Code Generator Pass Configuration Options.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
operand_type_match m_Pred()
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Or< Preds... > m_any_of(Preds &&... preds)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
void initializeAArch64PostLegalizerCombinerPass(PassRegistry &)
std::optional< APInt > isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a splat vector of constant integers.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
FunctionPass * createAArch64PostLegalizerCombiner(bool IsOptNone)
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...