LLVM 19.0.0git
Macros | Functions | Variables
AArch64LoadStoreOptimizer.cpp File Reference
#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <optional>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "aarch64-ldst-opt"
 
#define AARCH64_LOAD_STORE_OPT_NAME   "AArch64 load / store optimization pass"
 

Functions

 STATISTIC (NumPairCreated, "Number of load/store pair instructions generated")
 
 STATISTIC (NumPostFolded, "Number of post-index updates folded")
 
 STATISTIC (NumPreFolded, "Number of pre-index updates folded")
 
 STATISTIC (NumUnscaledPairCreated, "Number of load/store from unscaled generated")
 
 STATISTIC (NumZeroStoresPromoted, "Number of narrow zero stores promoted")
 
 STATISTIC (NumLoadsFromStoresPromoted, "Number of loads from stores promoted")
 
 STATISTIC (NumFailedAlignmentCheck, "Number of load/store pair transformation " "not passed the alignment check")
 
 DEBUG_COUNTER (RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming")
 
 INITIALIZE_PASS (AArch64LoadStoreOpt, "aarch64-ldst-opt", AARCH64_LOAD_STORE_OPT_NAME, false, false) static bool isNarrowStore(unsigned Opc)
 
static bool isTagStore (const MachineInstr &MI)
 
static unsigned getMatchingNonSExtOpcode (unsigned Opc, bool *IsValidLdStrOpc=nullptr)
 
static unsigned getMatchingWideOpcode (unsigned Opc)
 
static unsigned getMatchingPairOpcode (unsigned Opc)
 
static unsigned isMatchingStore (MachineInstr &LoadInst, MachineInstr &StoreInst)
 
static unsigned getPreIndexedOpcode (unsigned Opc)
 
static unsigned getPostIndexedOpcode (unsigned Opc)
 
static bool isPreLdStPairCandidate (MachineInstr &FirstMI, MachineInstr &MI)
 
static void getPrePostIndexedMemOpInfo (const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
 
static MachineOperandgetLdStRegOp (MachineInstr &MI, unsigned PairedRegOp=0)
 
static bool isLdOffsetInRangeOfSt (MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
 
static bool isPromotableZeroStoreInst (MachineInstr &MI)
 
static bool isPromotableLoadFromStore (MachineInstr &MI)
 
static bool isMergeableLdStUpdate (MachineInstr &MI)
 
static bool isRewritableImplicitDef (unsigned Opc)
 
static bool forAllMIsUntilDef (MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
 
static void updateDefinedRegisters (MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
 
static bool inBoundsForPair (bool IsUnscaled, int Offset, int OffsetStride)
 
static int alignTo (int Num, int PowOf2)
 
static bool mayAlias (MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
 
static bool needsWinCFI (const MachineFunction *MF)
 
static bool areCandidatesToMergeOrPair (MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
 
static bool canRenameMOP (const MachineOperand &MOP, const TargetRegisterInfo *TRI)
 
static bool canRenameUpToDef (MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
 
static bool canRenameUntilSecondLoad (MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
 
static std::optional< MCPhysRegtryToFindRegisterToRename (const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
 
static std::optional< MCPhysRegfindRenameRegForSameLdStRegPair (std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
 
static MachineBasicBlock::iterator maybeMoveCFI (MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
 

Variables

static cl::opt< unsignedLdStLimit ("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
 
static cl::opt< unsignedUpdateLimit ("aarch64-update-scan-limit", cl::init(100), cl::Hidden)
 
static cl::opt< boolEnableRenaming ("aarch64-load-store-renaming", cl::init(true), cl::Hidden)
 

Macro Definition Documentation

◆ AARCH64_LOAD_STORE_OPT_NAME

#define AARCH64_LOAD_STORE_OPT_NAME   "AArch64 load / store optimization pass"

Definition at line 84 of file AArch64LoadStoreOptimizer.cpp.

◆ DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-ldst-opt"

Definition at line 56 of file AArch64LoadStoreOptimizer.cpp.

Function Documentation

◆ alignTo()

static int alignTo ( int  Num,
int  PowOf2 
)
static

Definition at line 1275 of file AArch64LoadStoreOptimizer.cpp.

◆ areCandidatesToMergeOrPair()

static bool areCandidatesToMergeOrPair ( MachineInstr FirstMI,
MachineInstr MI,
LdStPairFlags &  Flags,
const AArch64InstrInfo TII 
)
static

◆ canRenameMOP()

static bool canRenameMOP ( const MachineOperand MOP,
const TargetRegisterInfo TRI 
)
static

◆ canRenameUntilSecondLoad()

static bool canRenameUntilSecondLoad ( MachineInstr FirstLoad,
MachineInstr SecondLoad,
LiveRegUnits UsedInBetween,
SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses,
const TargetRegisterInfo TRI 
)
static

◆ canRenameUpToDef()

static bool canRenameUpToDef ( MachineInstr FirstMI,
LiveRegUnits UsedInBetween,
SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses,
const TargetRegisterInfo TRI 
)
static

◆ DEBUG_COUNTER()

DEBUG_COUNTER ( RegRenamingCounter  ,
DEBUG_TYPE "-reg-renaming"  ,
"Controls which pairs are considered for renaming"   
)

◆ findRenameRegForSameLdStRegPair()

static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair ( std::optional< bool MaybeCanRename,
MachineInstr FirstMI,
MachineInstr MI,
Register  Reg,
LiveRegUnits DefinedInBB,
LiveRegUnits UsedInBetween,
SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses,
const TargetRegisterInfo TRI 
)
static

◆ forAllMIsUntilDef()

static bool forAllMIsUntilDef ( MachineInstr MI,
MCPhysReg  DefReg,
const TargetRegisterInfo TRI,
unsigned  Limit,
std::function< bool(MachineInstr &, bool)> &  Fn 
)
static

◆ getLdStRegOp()

static MachineOperand & getLdStRegOp ( MachineInstr MI,
unsigned  PairedRegOp = 0 
)
static

◆ getMatchingNonSExtOpcode()

static unsigned getMatchingNonSExtOpcode ( unsigned  Opc,
bool IsValidLdStrOpc = nullptr 
)
static

Definition at line 249 of file AArch64LoadStoreOptimizer.cpp.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingPairOpcode()

static unsigned getMatchingPairOpcode ( unsigned  Opc)
static

Definition at line 321 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingWideOpcode()

static unsigned getMatchingWideOpcode ( unsigned  Opc)
static

Definition at line 302 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPostIndexedOpcode()

static unsigned getPostIndexedOpcode ( unsigned  Opc)
static

Definition at line 486 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPreIndexedOpcode()

static unsigned getPreIndexedOpcode ( unsigned  Opc)
static

Definition at line 413 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPrePostIndexedMemOpInfo()

static void getPrePostIndexedMemOpInfo ( const MachineInstr MI,
int &  Scale,
int &  MinOffset,
int &  MaxOffset 
)
static

◆ inBoundsForPair()

static bool inBoundsForPair ( bool  IsUnscaled,
int  Offset,
int  OffsetStride 
)
static

Definition at line 1258 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Offset.

◆ INITIALIZE_PASS()

INITIALIZE_PASS ( AArch64LoadStoreOpt  ,
"aarch64-ldst-opt"  ,
AARCH64_LOAD_STORE_OPT_NAME  ,
false  ,
false   
)

Definition at line 220 of file AArch64LoadStoreOptimizer.cpp.

◆ isLdOffsetInRangeOfSt()

static bool isLdOffsetInRangeOfSt ( MachineInstr LoadInst,
MachineInstr StoreInst,
const AArch64InstrInfo TII 
)
static

◆ isMatchingStore()

static unsigned isMatchingStore ( MachineInstr LoadInst,
MachineInstr StoreInst 
)
static

◆ isMergeableLdStUpdate()

static bool isMergeableLdStUpdate ( MachineInstr MI)
static

◆ isPreLdStPairCandidate()

static bool isPreLdStPairCandidate ( MachineInstr FirstMI,
MachineInstr MI 
)
static

Definition at line 565 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOpcode(), and MI.

Referenced by areCandidatesToMergeOrPair().

◆ isPromotableLoadFromStore()

static bool isPromotableLoadFromStore ( MachineInstr MI)
static

Definition at line 653 of file AArch64LoadStoreOptimizer.cpp.

References MI.

◆ isPromotableZeroStoreInst()

static bool isPromotableZeroStoreInst ( MachineInstr MI)
static

Definition at line 646 of file AArch64LoadStoreOptimizer.cpp.

References getLdStRegOp(), llvm::MachineOperand::getReg(), and MI.

◆ isRewritableImplicitDef()

static bool isRewritableImplicitDef ( unsigned  Opc)
static

Definition at line 727 of file AArch64LoadStoreOptimizer.cpp.

Referenced by canRenameMOP().

◆ isTagStore()

static bool isTagStore ( const MachineInstr MI)
static

Definition at line 237 of file AArch64LoadStoreOptimizer.cpp.

References MI.

Referenced by getPrePostIndexedMemOpInfo().

◆ mayAlias()

static bool mayAlias ( MachineInstr MIa,
SmallVectorImpl< MachineInstr * > &  MemInsns,
AliasAnalysis AA 
)
static

◆ maybeMoveCFI()

static MachineBasicBlock::iterator maybeMoveCFI ( MachineInstr MI,
MachineBasicBlock::iterator  MaybeCFI 
)
static

◆ needsWinCFI()

static bool needsWinCFI ( const MachineFunction MF)
static

◆ STATISTIC() [1/7]

STATISTIC ( NumFailedAlignmentCheck  ,
"Number of load/store pair transformation " "not passed the alignment check"   
)

◆ STATISTIC() [2/7]

STATISTIC ( NumLoadsFromStoresPromoted  ,
"Number of loads from stores promoted"   
)

◆ STATISTIC() [3/7]

STATISTIC ( NumPairCreated  ,
"Number of load/store pair instructions generated"   
)

◆ STATISTIC() [4/7]

STATISTIC ( NumPostFolded  ,
"Number of post-index updates folded"   
)

◆ STATISTIC() [5/7]

STATISTIC ( NumPreFolded  ,
"Number of pre-index updates folded"   
)

◆ STATISTIC() [6/7]

STATISTIC ( NumUnscaledPairCreated  ,
"Number of load/store from unscaled generated"   
)

◆ STATISTIC() [7/7]

STATISTIC ( NumZeroStoresPromoted  ,
"Number of narrow zero stores promoted"   
)

◆ tryToFindRegisterToRename()

static std::optional< MCPhysReg > tryToFindRegisterToRename ( const MachineFunction MF,
Register  Reg,
LiveRegUnits DefinedInBB,
LiveRegUnits UsedInBetween,
SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses,
const TargetRegisterInfo TRI 
)
static

◆ updateDefinedRegisters()

static void updateDefinedRegisters ( MachineInstr MI,
LiveRegUnits Units,
const TargetRegisterInfo TRI 
)
static

Variable Documentation

◆ EnableRenaming

cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden) ( "aarch64-load-store-renaming"  ,
cl::init(true ,
cl::Hidden   
)
static

◆ LdStLimit

cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden) ( "aarch64-load-store-scan-limit"  ,
cl::init(20)  ,
cl::Hidden   
)
static

Referenced by canRenameUpToDef().

◆ UpdateLimit

cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden) ( "aarch64-update-scan-limit"  ,
cl::init(100)  ,
cl::Hidden   
)
static