AArch64TargetTransformInfo.cpp File Reference

#include "AArch64TargetTransformInfo.h"
#include "AArch64ExpandImm.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64SMEAttributes.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
#include "llvm/CodeGen/CostTable.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <optional>

Go to the source code of this file.

Classes
struct	SVEIntrinsicInfo

Macros
#define	DEBUG_TYPE   "aarch64tti"

Functions
static bool	isSMEABIRoutineCall (const CallInst &CI, const AArch64TargetLowering &TLI)
static bool	hasPossibleIncompatibleOps (const Function *F, const AArch64TargetLowering &TLI)
	Returns true if the function has explicit operations that can only be lowered using incompatible instructions for the selected mode.
static bool	isUnpackedVectorVT (EVT VecVT)
static InstructionCost	getHistogramCost (const AArch64Subtarget *ST, const IntrinsicCostAttributes &ICA)
static std::optional< Instruction * >	processPhiNode (InstCombiner &IC, IntrinsicInst &II)
	The function will remove redundant reinterprets casting in the presence of the control flow.
static SVEIntrinsicInfo	constructSVEIntrinsicInfo (IntrinsicInst &II)
static bool	isAllActivePredicate (Value *Pred)
static Value *	stripInactiveLanes (Value *V, const Value *Pg)
static std::optional< Instruction * >	simplifySVEIntrinsicBinOp (InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * >	simplifySVEIntrinsic (InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * >	tryCombineFromSVBoolBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineConvertFromSVBool (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESel (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEDup (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEDupX (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECmpNE (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVELast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECondLast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineRDFFR (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECntElts (InstCombiner &IC, IntrinsicInst &II, unsigned NumElts)
static std::optional< Instruction * >	instCombineSMECntsd (InstCombiner &IC, IntrinsicInst &II, const AArch64Subtarget *ST)
static std::optional< Instruction * >	instCombineSVEPTest (InstCombiner &IC, IntrinsicInst &II)
template<Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
static std::optional< Instruction * >	instCombineSVEVectorFuseMulAddSub (InstCombiner &IC, IntrinsicInst &II, bool MergeIntoAddendOp)
static std::optional< Instruction * >	instCombineSVELD1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static std::optional< Instruction * >	instCombineSVEST1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static Instruction::BinaryOps	intrinsicIDToBinOpCode (unsigned Intrinsic)
static std::optional< Instruction * >	instCombineSVEVectorBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFAddU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFSubU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEUnpack (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVETBL (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEUzp1 (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEZip (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineLD1GatherIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineST1ScatterIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESDIV (InstCombiner &IC, IntrinsicInst &II)
bool	SimplifyValuePattern (SmallVector< Value * > &Vec, bool AllowPoison)
static std::optional< Instruction * >	instCombineSVEDupqLane (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineMaxMinNM (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESrshl (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEInsr (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineDMB (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineWhilelo (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombinePTrue (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEUxt (InstCombiner &IC, IntrinsicInst &II, unsigned NumBits)
static std::optional< Instruction * >	instCombineInStreamingMode (InstCombiner &IC, IntrinsicInst &II)
static unsigned	getSVEGatherScatterOverhead (unsigned Opcode, const AArch64Subtarget *ST)
static void	getFalkorUnrollingPreferences (Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP)
static bool	isLoopSizeWithinBudget (Loop *L, const AArch64TTIImpl &TTI, InstructionCost Budget, unsigned *FinalSize)
static bool	shouldUnrollMultiExitLoop (Loop *L, ScalarEvolution &SE, const AArch64TTIImpl &TTI)
static void	getAppleRuntimeUnrollPreferences (Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP, const AArch64TTIImpl &TTI)
	For Apple CPUs, we want to runtime-unroll loops to make better use if the OOO engine's wide instruction window and various predictors.
static bool	containsDecreasingPointers (Loop *TheLoop, PredicatedScalarEvolution *PSE, const DominatorTree &DT)
static bool	isSplatShuffle (Value *V)
static bool	areExtractShuffleVectors (Value *Op1, Value *Op2, bool AllowSplat=false)
	Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.
static bool	areExtractExts (Value *Ext1, Value *Ext2)
	Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static bool	isOperandOfVmullHighP64 (Value *Op)
	Check if Op could be used with vmull_high_p64 intrinsic.
static bool	areOperandsOfVmullHighP64 (Value *Op1, Value *Op2)
	Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
static bool	shouldSinkVectorOfPtrs (Value *Ptrs, SmallVectorImpl< Use * > &Ops)
static bool	shouldSinkVScale (Value *Op, SmallVectorImpl< Use * > &Ops)
	We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.

Variables
static cl::opt< bool >	EnableFalkorHWPFUnrollFix ("enable-falkor-hwpf-unroll-fix", cl::init(true), cl::Hidden)
static cl::opt< bool >	SVEPreferFixedOverScalableIfEqualCost ("sve-prefer-fixed-over-scalable-if-equal", cl::Hidden)
static cl::opt< unsigned >	SVEGatherOverhead ("sve-gather-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	SVEScatterOverhead ("sve-scatter-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	SVETailFoldInsnThreshold ("sve-tail-folding-insn-threshold", cl::init(15), cl::Hidden)
static cl::opt< unsigned >	NeonNonConstStrideOverhead ("neon-nonconst-stride-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	CallPenaltyChangeSM ("call-penalty-sm-change", cl::init(5), cl::Hidden, cl::desc("Penalty of calling a function that requires a change to PSTATE.SM"))
static cl::opt< unsigned >	InlineCallPenaltyChangeSM ("inline-call-penalty-sm-change", cl::init(10), cl::Hidden, cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM"))
static cl::opt< bool >	EnableOrLikeSelectOpt ("enable-aarch64-or-like-select", cl::init(true), cl::Hidden)
static cl::opt< bool >	EnableLSRCostOpt ("enable-aarch64-lsr-cost-opt", cl::init(true), cl::Hidden)
static cl::opt< unsigned >	BaseHistCntCost ("aarch64-base-histcnt-cost", cl::init(8), cl::Hidden, cl::desc("The cost of a histcnt instruction"))
static cl::opt< unsigned >	DMBLookaheadThreshold ("dmb-lookahead-threshold", cl::init(10), cl::Hidden, cl::desc("The number of instructions to search for a redundant dmb"))
static cl::opt< int >	Aarch64ForceUnrollThreshold ("aarch64-force-unroll-threshold", cl::init(0), cl::Hidden, cl::desc("Threshold for forced unrolling of small loops in AArch64"))
TailFoldingOption	TailFoldingOptionLoc
static cl::opt< TailFoldingOption, true, cl::parser< std::string > >	SVETailFolding ("sve-tail-folding", cl::desc("Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc))
static cl::opt< bool >	EnableFixedwidthAutovecInStreamingMode ("enable-fixedwidth-autovec-in-streaming-mode", cl::init(false), cl::Hidden)
static cl::opt< bool >	EnableScalableAutovecInStreamingMode ("enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64tti"

Definition at line 35 of file AArch64TargetTransformInfo.cpp.

Function Documentation

areExtractExts()

bool areExtractExts ( Value * Ext1, Value * Ext2 )

static

Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.

Definition at line 6453 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExtOrSExt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands(), and llvm::ARMTTIImpl::isProfitableToSinkOperands().

areExtractShuffleVectors()

bool areExtractShuffleVectors ( Value * Op1, Value * Op2, bool AllowSplat = false )

static

Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.

Definition at line 6393 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm::ShuffleVectorInst::isExtractSubvectorMask(), llvm::Type::isScalableTy(), isSplatShuffle(), llvm::M1(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

areOperandsOfVmullHighP64()

bool areOperandsOfVmullHighP64 ( Value * Op1, Value * Op2 )

static

Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.

Definition at line 6480 of file AArch64TargetTransformInfo.cpp.

References isOperandOfVmullHighP64().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

constructSVEIntrinsicInfo()

SVEIntrinsicInfo constructSVEIntrinsicInfo ( IntrinsicInst & II )

static

Definition at line 1311 of file AArch64TargetTransformInfo.cpp.

References llvm::all_of(), SVEIntrinsicInfo::defaultMergingOp(), SVEIntrinsicInfo::defaultMergingUnaryNarrowingTopOp(), SVEIntrinsicInfo::defaultMergingUnaryOp(), SVEIntrinsicInfo::defaultUndefOp(), SVEIntrinsicInfo::defaultVoidOp(), SVEIntrinsicInfo::defaultZeroingOp(), II, llvm::isa(), and SVEIntrinsicInfo::setMatchingIROpcode().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

containsDecreasingPointers()

bool containsDecreasingPointers ( Loop * TheLoop, PredicatedScalarEvolution * PSE, const DominatorTree & DT )

static

Definition at line 6247 of file AArch64TargetTransformInfo.cpp.

References llvm::LoopBase< BlockT, LoopT >::blocks(), llvm::getLoadStorePointerOperand(), llvm::getLoadStoreType(), llvm::getPtrStride(), I, and llvm::isa().

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().

getAppleRuntimeUnrollPreferences()

void getAppleRuntimeUnrollPreferences ( Loop * L, ScalarEvolution & SE, TargetTransformInfo::UnrollingPreferences & UP, const AArch64TTIImpl & TTI )

static

For Apple CPUs, we want to runtime-unroll loops to make better use if the OOO engine's wide instruction window and various predictors.

Definition at line 5111 of file AArch64TargetTransformInfo.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AddAdditionalAccumulators, llvm::any_of(), llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::TargetTransformInfo::UnrollingPreferences::DefaultUnrollRuntimeCount, llvm::Depth, llvm::dyn_cast(), llvm::findStringMetadataForLoop(), llvm::ScalarEvolution::getBackedgeTakenCount(), llvm::getLoadStorePointerOperand(), llvm::ScalarEvolution::getSCEV(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSmallConstantTripMultiple(), llvm::ScalarEvolution::getSymbolicMaxBackedgeTakenCount(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::is_contained(), llvm::isa(), llvm::ScalarEvolution::isLoopInvariant(), isLoopSizeWithinBudget(), llvm::PatternMatch::m_Br(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Instruction(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::TargetTransformInfo::UnrollingPreferences::MaxCount, llvm::none_of(), llvm::TargetTransformInfo::UnrollingPreferences::Partial, llvm::predecessors(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::SCEVExpansionBudget, Size, and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

getFalkorUnrollingPreferences()

void getFalkorUnrollingPreferences ( Loop * L, ScalarEvolution & SE, TargetTransformInfo::UnrollingPreferences & UP )

static

Definition at line 4996 of file AArch64TargetTransformInfo.cpp.

References llvm::dbgs(), llvm::dyn_cast(), llvm::LoadInst::getPointerOperand(), llvm::ScalarEvolution::getSCEV(), I, llvm::SCEVAddRecExpr::isAffine(), LLVM_DEBUG, llvm::Log2_32(), and llvm::TargetTransformInfo::UnrollingPreferences::MaxCount.

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

getHistogramCost()

InstructionCost getHistogramCost ( const AArch64Subtarget * ST, const IntrinsicCostAttributes & ICA )

static

Definition at line 565 of file AArch64TargetTransformInfo.cpp.

References BaseHistCntCost, llvm::dyn_cast(), llvm::IntrinsicCostAttributes::getArgTypes(), llvm::InstructionCost::getInvalid(), llvm::Type::getScalarSizeInBits(), InstructionCost, llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm::isPowerOf2_64(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

getSVEGatherScatterOverhead()

unsigned getSVEGatherScatterOverhead ( unsigned Opcode, const AArch64Subtarget * ST )

static

Definition at line 4777 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm_unreachable, SVEGatherOverhead, and SVEScatterOverhead.

Referenced by llvm::AArch64TTIImpl::getGatherScatterOpCost().

hasPossibleIncompatibleOps()

bool hasPossibleIncompatibleOps ( const Function * F, const AArch64TargetLowering & TLI )

static

Returns true if the function has explicit operations that can only be lowered using incompatible instructions for the selected mode.

This also returns true if the function F may use or modify ZA state.

Definition at line 238 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), F, I, llvm::isa(), and isSMEABIRoutineCall().

Referenced by llvm::AArch64TTIImpl::areInlineCompatible().

instCombineConvertFromSVBool()

std::optional< Instruction * > instCombineConvertFromSVBool ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1828 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::dyn_cast(), II, llvm::SmallVectorImpl< T >::insert(), llvm::isa(), processPhiNode(), llvm::InstCombiner::replaceInstUsesWith(), and tryCombineFromSVBoolBinOp().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineDMB()

std::optional< Instruction * > instCombineDMB ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2789 of file AArch64TargetTransformInfo.cpp.

References DMBLookaheadThreshold, llvm::dyn_cast_or_null(), llvm::InstCombiner::eraseInstFromFunction(), I, and II.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineInStreamingMode()

std::optional< Instruction * > instCombineInStreamingMode ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2851 of file AArch64TargetTransformInfo.cpp.

References llvm::ConstantInt::getBool(), llvm::SMEAttrs::hasStreamingCompatibleInterface(), llvm::SMEAttrs::hasStreamingInterfaceOrBody(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineLD1GatherIndex()

std::optional< Instruction * > instCombineLD1GatherIndex ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2552 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedLoad(), llvm::ConstantAggregateZero::get(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineMaxMinNM()

std::optional< Instruction * > instCombineMaxMinNM ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2735 of file AArch64TargetTransformInfo.cpp.

References A(), B(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombinePTrue()

std::optional< Instruction * > instCombinePTrue ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2823 of file AArch64TargetTransformInfo.cpp.

References llvm::Constant::getAllOnesValue(), II, llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineRDFFR()

std::optional< Instruction * > instCombineRDFFR ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2135 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::Type::getInt32Ty(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSMECntsd()

std::optional< Instruction * > instCombineSMECntsd ( InstCombiner & IC, IntrinsicInst & II, const AArch64Subtarget * ST )

static

Definition at line 2170 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateElementCount(), llvm::ElementCount::getScalable(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineST1ScatterIndex()

std::optional< Instruction * > instCombineST1ScatterIndex ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2580 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getType(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVECmpNE()

std::optional< Instruction * > instCombineSVECmpNE ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1926 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::ScalableVectorType::get(), llvm::Type::getInt1Ty(), llvm::Type::getInt32Ty(), llvm::Constant::getNullValue(), llvm::getSplatValue(), I, II, llvm::isa(), isAllActivePredicate(), llvm::InstCombiner::replaceInstUsesWith(), llvm::AArch64::SVEBitsPerBlock, and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVECntElts()

std::optional< Instruction * > instCombineSVECntElts ( InstCombiner & IC, IntrinsicInst & II, unsigned NumElts )

static

Definition at line 2151 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateElementCount(), llvm::getNumElementsFromSVEPredPattern(), llvm::ElementCount::getScalable(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVECondLast()

std::optional< Instruction * > instCombineSVECondLast ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2093 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateIntrinsic(), Fallback, llvm::VectorType::get(), getBitWidth(), llvm::IRBuilderBase::getDoubleTy(), llvm::IRBuilderBase::getFloatTy(), llvm::IRBuilderBase::getHalfTy(), llvm::Value::getType(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDup()

std::optional< Instruction * > instCombineSVEDup ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1892 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::InsertElementInst::Create(), llvm::dyn_cast(), llvm::Type::getInt64Ty(), llvm::IntrinsicInst::getIntrinsicID(), llvm::User::getOperand(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDupqLane()

std::optional< Instruction * > instCombineSVEDupqLane ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2673 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateBitOrPointerCast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateInsertVector(), llvm::IRBuilderBase::CreateShuffleVector(), Default, llvm::dyn_cast(), llvm::ConstantAggregateZero::get(), llvm::PoisonValue::get(), llvm::ScalableVectorType::get(), llvm::IRBuilderBase::getInt32Ty(), llvm::IRBuilderBase::getInt64(), llvm::IRBuilderBase::getIntNTy(), llvm::Value::getType(), I, II, llvm::isa(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), SimplifyValuePattern(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDupX()

std::optional< Instruction * > instCombineSVEDupX ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1916 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateVectorSplat(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Splat.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEInsr()

std::optional< Instruction * > instCombineSVEInsr ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2779 of file AArch64TargetTransformInfo.cpp.

References llvm::getSplatValue(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVELast()

std::optional< Instruction * > instCombineSVELast ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2019 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::CallingConv::C, llvm::cast(), llvm::ExtractElementInst::Create(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::BinaryOperator::CreateWithCopiedFlags(), llvm::dyn_cast(), llvm::Type::getInt64Ty(), llvm::getNumElementsFromSVEPredPattern(), llvm::getSplatValue(), llvm::Value::getType(), II, llvm::isSplatValue(), LHS, llvm::PatternMatch::m_BinOp(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and RHS.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVELD1()

std::optional< Instruction * > instCombineSVELD1 ( InstCombiner & IC, IntrinsicInst & II, const DataLayout & DL )

static

Definition at line 2296 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateLoad(), llvm::IRBuilderBase::CreateMaskedLoad(), DL, llvm::ConstantAggregateZero::get(), llvm::Value::getPointerAlignment(), II, isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEPTest()

std::optional< Instruction * > instCombineSVEPTest ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2183 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::IntrinsicInst::getIntrinsicID(), llvm::Value::getType(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESDIV()

std::optional< Instruction * > instCombineSVESDIV ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2606 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast_or_null(), llvm::IRBuilderBase::getInt32Ty(), llvm::APInt::getSExtValue(), llvm::getSplatValue(), llvm::ConstantInt::getValue(), II, llvm::Int32Ty, llvm::APInt::isNegatedPowerOf2(), llvm::APInt::isPowerOf2(), llvm::APInt::logBase2(), llvm::APInt::negate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESel()

std::optional< Instruction * > instCombineSVESel ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1880 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateSelect(), II, isAllActivePredicate(), llvm::InstCombiner::replaceInstUsesWith(), and Select.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESrshl()

std::optional< Instruction * > instCombineSVESrshl ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2745 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::isa(), isAllActivePredicate(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_NonNegative(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEST1()

std::optional< Instruction * > instCombineSVEST1 ( InstCombiner & IC, IntrinsicInst & II, const DataLayout & DL )

static

Definition at line 2315 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::IRBuilderBase::CreateStore(), DL, llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getPointerAlignment(), II, and isAllActivePredicate().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVETBL()

std::optional< Instruction * > instCombineSVETBL ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2483 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::dyn_cast_or_null(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEUnpack()

std::optional< Instruction * > instCombineSVEUnpack ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2463 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntCast(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEUxt()

std::optional< Instruction * > instCombineSVEUxt ( InstCombiner & IC, IntrinsicInst & II, unsigned NumBits )

static

Definition at line 2830 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::APInt::getLowBitsSet(), II, llvm::isa(), isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEUzp1()

std::optional< Instruction * > instCombineSVEUzp1 ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2506 of file AArch64TargetTransformInfo.cpp.

References A(), B(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateInsertVector(), llvm::PoisonValue::get(), llvm::ScalableVectorType::getDoubleElementsVectorType(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorAdd()

std::optional< Instruction * > instCombineSVEVectorAdd ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2361 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorBinOp()

std::optional< Instruction * > instCombineSVEVectorBinOp ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2346 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBinOpFMF(), II, intrinsicIDToBinOpCode(), isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorFAddU(), and instCombineSVEVectorFSubU().

instCombineSVEVectorFAdd()

std::optional< Instruction * > instCombineSVEVectorFAdd ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2375 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFAddU()

std::optional< Instruction * > instCombineSVEVectorFAddU ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2395 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFSub()

std::optional< Instruction * > instCombineSVEVectorFSub ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2415 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFSubU()

std::optional< Instruction * > instCombineSVEVectorFSubU ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2435 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFuseMulAddSub()

template<Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>

std::optional< Instruction * > instCombineSVEVectorFuseMulAddSub ( InstCombiner & IC, IntrinsicInst & II, bool MergeIntoAddendOp )

static

Definition at line 2253 of file AArch64TargetTransformInfo.cpp.

References llvm::FastMathFlags::allowContract(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), Mul, P, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorAdd(), instCombineSVEVectorFAdd(), instCombineSVEVectorFAddU(), instCombineSVEVectorFSub(), instCombineSVEVectorFSubU(), and instCombineSVEVectorSub().

instCombineSVEVectorSub()

std::optional< Instruction * > instCombineSVEVectorSub ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2454 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEZip()

std::optional< Instruction * > instCombineSVEZip ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2536 of file AArch64TargetTransformInfo.cpp.

References A(), B(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineWhilelo()

std::optional< Instruction * > instCombineWhilelo ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 2814 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

intrinsicIDToBinOpCode()

Instruction::BinaryOps intrinsicIDToBinOpCode ( unsigned Intrinsic )

static

Definition at line 2332 of file AArch64TargetTransformInfo.cpp.

Referenced by instCombineSVEVectorBinOp().

isAllActivePredicate()

bool isAllActivePredicate ( Value * Pred )

static

Definition at line 1635 of file AArch64TargetTransformInfo.cpp.

References llvm::CallingConv::C, llvm::cast(), llvm::dyn_cast(), llvm::Value::getType(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

isLoopSizeWithinBudget()

bool isLoopSizeWithinBudget ( Loop * L, const AArch64TTIImpl & TTI, InstructionCost Budget, unsigned * FinalSize )

static

Definition at line 5047 of file AArch64TargetTransformInfo.cpp.

References llvm::InstructionCost::getValue(), I, and llvm::TargetTransformInfo::TCK_CodeSize.

Referenced by getAppleRuntimeUnrollPreferences(), and shouldUnrollMultiExitLoop().

isOperandOfVmullHighP64()

bool isOperandOfVmullHighP64 ( Value * Op )

static

Check if Op could be used with vmull_high_p64 intrinsic.

Definition at line 6469 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::Value::getType(), llvm::ConstantInt::getValue(), llvm::isa(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by areOperandsOfVmullHighP64().

isSMEABIRoutineCall()

bool isSMEABIRoutineCall ( const CallInst & CI, const AArch64TargetLowering & TLI )

static

Definition at line 228 of file AArch64TargetTransformInfo.cpp.

References F, llvm::CallBase::getCalledFunction(), llvm::TargetLoweringBase::getRuntimeLibcallsInfo(), and llvm::SMEAttrs::isSMEABIRoutine().

Referenced by hasPossibleIncompatibleOps().

isSplatShuffle()

bool isSplatShuffle ( Value * V )

static

Definition at line 6385 of file AArch64TargetTransformInfo.cpp.

References llvm::all_equal(), and llvm::dyn_cast().

Referenced by areExtractShuffleVectors(), and llvm::AArch64TTIImpl::isProfitableToSinkOperands().

isUnpackedVectorVT()

bool isUnpackedVectorVT ( EVT VecVT )

static

Definition at line 560 of file AArch64TargetTransformInfo.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::EVT::isScalableVector(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

processPhiNode()

std::optional< Instruction * > processPhiNode ( InstCombiner & IC, IntrinsicInst & II )

static

The function will remove redundant reinterprets casting in the presence of the control flow.

Definition at line 1071 of file AArch64TargetTransformInfo.cpp.

References llvm::PHINode::addIncoming(), assert(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreatePHI(), llvm::dyn_cast(), I, II, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::SetInsertPoint().

Referenced by instCombineConvertFromSVBool().

shouldSinkVectorOfPtrs()

bool shouldSinkVectorOfPtrs ( Value * Ptrs, SmallVectorImpl< Use * > & Ops )

static

Definition at line 6484 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, llvm::cast(), llvm::dyn_cast(), GEP, and llvm::isa().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

shouldSinkVScale()

bool shouldSinkVScale ( Value * Op, SmallVectorImpl< Use * > & Ops )

static

We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.

Definition at line 6512 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::m_ZExt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

shouldUnrollMultiExitLoop()

bool shouldUnrollMultiExitLoop ( Loop * L, ScalarEvolution & SE, const AArch64TTIImpl & TTI )

static

Definition at line 5074 of file AArch64TargetTransformInfo.cpp.

References llvm::any_of(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSymbolicMaxBackedgeTakenCount(), llvm::BasicBlock::getTerminator(), llvm::isa(), isLoopSizeWithinBudget(), and llvm::ArrayRef< T >::size().

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

simplifySVEIntrinsic()

std::optional< Instruction * > simplifySVEIntrinsic ( InstCombiner & IC, IntrinsicInst & II, const SVEIntrinsicInfo & IInfo )

static

Definition at line 1724 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::eraseInstFromFunction(), llvm::UndefValue::get(), SVEIntrinsicInfo::getGoverningPredicateOperandIdx(), SVEIntrinsicInfo::getMatchingIROpode(), SVEIntrinsicInfo::getMatchingUndefIntrinsic(), llvm::Constant::getNullValue(), SVEIntrinsicInfo::getOperandIdxInactiveLanesTakenFrom(), SVEIntrinsicInfo::getOperandIdxWithNoActiveLanes(), llvm::Intrinsic::getOrInsertDeclaration(), SVEIntrinsicInfo::hasGoverningPredicate(), SVEIntrinsicInfo::hasMatchingIROpode(), SVEIntrinsicInfo::hasMatchingUndefIntrinsic(), SVEIntrinsicInfo::hasOperandWithNoActiveLanes(), II, SVEIntrinsicInfo::inactiveLanesAreUnused(), SVEIntrinsicInfo::inactiveLanesTakenFromOperand(), llvm::isa(), isAllActivePredicate(), llvm::Instruction::isBinaryOp(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), OpIdx, llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), SVEIntrinsicInfo::resultIsZeroInitialized(), and simplifySVEIntrinsicBinOp().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

simplifySVEIntrinsicBinOp()

std::optional< Instruction * > simplifySVEIntrinsicBinOp ( InstCombiner & IC, IntrinsicInst & II, const SVEIntrinsicInfo & IInfo )

static

Definition at line 1672 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateSelect(), DL, llvm::dyn_cast(), SVEIntrinsicInfo::getMatchingIROpode(), SVEIntrinsicInfo::getOperandIdxInactiveLanesTakenFrom(), II, SVEIntrinsicInfo::inactiveLanesAreNotDefined(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::Instruction::isCommutative(), Opc, llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), llvm::simplifyBinOp(), and stripInactiveLanes().

Referenced by simplifySVEIntrinsic().

SimplifyValuePattern()

bool SimplifyValuePattern	(	SmallVector< Value * > &	Vec,
		bool	AllowPoison )

Definition at line 2644 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::isPowerOf2_64(), LHS, llvm::SmallVectorImpl< T >::resize(), RHS, SimplifyValuePattern(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by instCombineSVEDupqLane(), and SimplifyValuePattern().

stripInactiveLanes()

Value * stripInactiveLanes ( Value * V, const Value * Pg )

static

Definition at line 1660 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::ConstantVector::getSplat(), and llvm::isa().

Referenced by simplifySVEIntrinsicBinOp().

tryCombineFromSVBoolBinOp()

std::optional< Instruction * > tryCombineFromSVBoolBinOp ( InstCombiner & IC, IntrinsicInst & II )

static

Definition at line 1779 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast(), II, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by instCombineConvertFromSVBool().

Variable Documentation

Aarch64ForceUnrollThreshold

cl::opt< int > Aarch64ForceUnrollThreshold("aarch64-force-unroll-threshold", cl::init(0), cl::Hidden, cl::desc("Threshold for forced unrolling of small loops in AArch64")) ( "aarch64-force-unroll-threshold" , cl::init(0) , cl::Hidden , cl::desc("Threshold for forced unrolling of small loops in AArch64")  )

static

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

BaseHistCntCost

cl::opt< unsigned > BaseHistCntCost("aarch64-base-histcnt-cost", cl::init(8), cl::Hidden, cl::desc("The cost of a histcnt instruction")) ( "aarch64-base-histcnt-cost" , cl::init(8) , cl::Hidden , cl::desc("The cost of a histcnt instruction")  )

static

Referenced by getHistogramCost().

CallPenaltyChangeSM

cl::opt< unsigned > CallPenaltyChangeSM("call-penalty-sm-change", cl::init(5), cl::Hidden, cl::desc( "Penalty of calling a function that requires a change to PSTATE.SM")) ( "call-penalty-sm-change" , cl::init(5) , cl::Hidden , cl::desc( "Penalty of calling a function that requires a change to PSTATE.SM")  )

static

Referenced by llvm::AArch64TTIImpl::getInlineCallPenalty().

DMBLookaheadThreshold

cl::opt< unsigned > DMBLookaheadThreshold("dmb-lookahead-threshold", cl::init(10), cl::Hidden, cl::desc("The number of instructions to search for a redundant dmb")) ( "dmb-lookahead-threshold" , cl::init(10) , cl::Hidden , cl::desc("The number of instructions to search for a redundant dmb")  )

static

Referenced by instCombineDMB().

EnableFalkorHWPFUnrollFix

cl::opt< bool > EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix", cl::init(true), cl::Hidden) ( "enable-falkor-hwpf-unroll-fix" , cl::init(true) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

EnableFixedwidthAutovecInStreamingMode

cl::opt< bool > EnableFixedwidthAutovecInStreamingMode("enable-fixedwidth-autovec-in-streaming-mode", cl::init(false), cl::Hidden) ( "enable-fixedwidth-autovec-in-streaming-mode" , cl::init(false) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::getRegisterBitWidth().

EnableLSRCostOpt

cl::opt< bool > EnableLSRCostOpt("enable-aarch64-lsr-cost-opt", cl::init(true), cl::Hidden) ( "enable-aarch64-lsr-cost-opt" , cl::init(true) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::isLSRCostLess().

EnableOrLikeSelectOpt

cl::opt< bool > EnableOrLikeSelectOpt("enable-aarch64-or-like-select", cl::init(true), cl::Hidden) ( "enable-aarch64-or-like-select" , cl::init(true) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::shouldTreatInstructionLikeSelect().

EnableScalableAutovecInStreamingMode

cl::opt< bool > EnableScalableAutovecInStreamingMode("enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden) ( "enable-scalable-autovec-in-streaming-mode" , cl::init(false) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::enableScalableVectorization(), and llvm::AArch64TTIImpl::getRegisterBitWidth().

InlineCallPenaltyChangeSM

cl::opt< unsigned > InlineCallPenaltyChangeSM("inline-call-penalty-sm-change", cl::init(10), cl::Hidden, cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM")) ( "inline-call-penalty-sm-change" , cl::init(10) , cl::Hidden , cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM")  )

static

Referenced by llvm::AArch64TTIImpl::getInlineCallPenalty().

NeonNonConstStrideOverhead

cl::opt< unsigned > NeonNonConstStrideOverhead("neon-nonconst-stride-overhead", cl::init(10), cl::Hidden) ( "neon-nonconst-stride-overhead" , cl::init(10) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::getAddressComputationCost().

SVEGatherOverhead

cl::opt< unsigned > SVEGatherOverhead("sve-gather-overhead", cl::init(10), cl::Hidden) ( "sve-gather-overhead" , cl::init(10) , cl::Hidden  )

static

Referenced by getSVEGatherScatterOverhead().

SVEPreferFixedOverScalableIfEqualCost

cl::opt< bool > SVEPreferFixedOverScalableIfEqualCost("sve-prefer-fixed-over-scalable-if-equal", cl::Hidden) ( "sve-prefer-fixed-over-scalable-if-equal" , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::preferFixedOverScalableIfEqualCost().

SVEScatterOverhead

cl::opt< unsigned > SVEScatterOverhead("sve-scatter-overhead", cl::init(10), cl::Hidden) ( "sve-scatter-overhead" , cl::init(10) , cl::Hidden  )

static

Referenced by getSVEGatherScatterOverhead().

SVETailFolding

cl::opt< TailFoldingOption, true, cl::parser< std::string > > SVETailFolding("sve-tail-folding", cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc)) ( "sve-tail-folding" , cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above") , cl::location(TailFoldingOptionLoc)  )

static

SVETailFoldInsnThreshold

cl::opt< unsigned > SVETailFoldInsnThreshold("sve-tail-folding-insn-threshold", cl::init(15), cl::Hidden) ( "sve-tail-folding-insn-threshold" , cl::init(15) , cl::Hidden  )

static

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().

TailFoldingOptionLoc

TailFoldingOption TailFoldingOptionLoc

Definition at line 191 of file AArch64TargetTransformInfo.cpp.

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().