SLPVectorizer.cpp File Reference

#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "SLPVectorizer/SLPCompatibilityAnalysis.h"
#include "SLPVectorizer/SLPCostAnalysis.h"
#include "SLPVectorizer/SLPMemoryUtils.h"
#include "SLPVectorizer/SLPReductionUtils.h"
#include "SLPVectorizer/SLPShuffleAnalysis.h"
#include "SLPVectorizer/SLPTypeUtils.h"
#include "SLPVectorizer/SLPUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	slpvectorizer::BoUpSLP
	Bottom Up SLP Vectorizer. More...
struct	slpvectorizer::BoUpSLP::StridedPtrInfo
	If we decide to generate strided load / store, this struct contains all the necessary info. More...
struct	slpvectorizer::BoUpSLP::EdgeInfo
	This structure holds any data we need about the edges being traversed during buildTreeRec(). More...
class	slpvectorizer::BoUpSLP::LookAheadHeuristics
	A helper class used for scoring candidates for two consecutive lanes. More...
class	slpvectorizer::BoUpSLP::VLOperands
	A helper data structure to hold the operands of a vector of instructions. More...
struct	llvm::DenseMapInfo< BoUpSLP::EdgeInfo >
struct	llvm::GraphTraits< BoUpSLP * >
struct	llvm::GraphTraits< BoUpSLP * >::ChildIteratorType
	Add the VectorizableTree to the index iterator to be able to return TreeEntry pointers. More...
class	llvm::GraphTraits< BoUpSLP * >::nodes_iterator
	For the node iterator we just need to turn the TreeEntry iterator into a TreeEntry* iterator so that it dereferences to NodeRef. More...
struct	llvm::DOTGraphTraits< BoUpSLP * >
class	slpvectorizer::BoUpSLP::ShuffleCostEstimator
	Merges shuffle masks and emits final shuffle instruction, if required. More...
class	slpvectorizer::BoUpSLP::ShuffleInstructionBuilder
	Merges shuffle masks and emits final shuffle instruction, if required. More...

Macros
#define	SV_NAME   "slp-vectorizer"
#define	DEBUG_TYPE   "SLP"

Functions
	STATISTIC (NumVectorInstructions, "Number of vector instructions generated")
	STATISTIC (NumStridedStoreChains, "Number of vectorized stride stores")
	STATISTIC (NumStoreChains, "Number of vector stores created")
	STATISTIC (NumVectorizedStores, "Number of vectorized stores")
	DEBUG_COUNTER (VectorizedGraphs, "slp-vectorized", "Controls which SLP graphs should be vectorized.")
static bool	areTwoInsertFromSameBuildVector (InsertElementInst *VU, InsertElementInst *V, function_ref< Value *(InsertElementInst *)> GetBaseOperand)
	Check if two insertelement instructions are from the same buildvector.
static LLVM_DUMP_METHOD void	dumpOrder (const BoUpSLP::OrdersType &Order)
static void	gatherPossiblyVectorizableLoads (const BoUpSLP &R, ArrayRef< Value * > VL, const DataLayout &DL, ScalarEvolution &SE, const TargetTransformInfo &TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > &GatheredLoads, bool AddNew=true)
	Tries to find subvector of loads and builds new vector of only loads if can be profitable.
static std::pair< size_t, size_t >	generateKeySubkey (Value *V, const TargetLibraryInfo *TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate)
	Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.
static bool	isMainInstruction (Instruction *I, Instruction *MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.
static SmallVector< Type * >	buildIntrinsicArgTypes (const CallInst *CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo *TTI)
	Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.
static std::pair< InstructionCost, InstructionCost >	getVectorCallCosts (CallInst *CI, Type *VecTy, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, ArrayRef< Type * > ArgTys, const TTI::TargetCostKind CostKind)
	Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.
static InstructionCost	getVectorOpCost (Instruction *I, unsigned VF, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const TTI::TargetCostKind CostKind)
static unsigned	getPoorThroughputTypeKey (Type *Ty)
	Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.
static bool	isPoorThroughputOp (Instruction *I, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, PoorThroughputOpCache &Cache, const TTI::TargetCostKind CostKind)
	Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.
static const Loop *	findInnermostNonInvariantLoop (const Loop *L, ArrayRef< Value * > VL)
	Find the innermost loop starting from L, for which at least a single value in VL is not invariant.
static bool	checkEVsForVecCalls (ArrayRef< Value * > VL, const InstructionsState &S, const TargetLibraryInfo &TLI, SmallVectorImpl< unsigned > &Indices, SmallVectorImpl< Value * > &Calls)
	Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.
static bool	allStructUsersAreExtractValueInsts (ArrayRef< Value * > VL)
static std::pair< Instruction *, Instruction * >	getMainAltOpsNoStateVL (ArrayRef< Value * > VL)
	Returns main/alternate instructions for the given VL.
static bool	tryToFindDuplicates (SmallVectorImpl< Value * > &VL, SmallVectorImpl< int > &ReuseShuffleIndices, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const InstructionsState &S, const BoUpSLP::EdgeInfo &UserTreeIdx, const BoUpSLP &R, bool BuildGatherOnly=true)
	Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.
static void	scanAssociativeOperands (const InstructionsState &S, DominatorTree &DT, const DataLayout &DL, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const BoUpSLP &R, SmallVectorImpl< BoUpSLP::ValueList > &Operands, SmallBitVector &NegatedColumns, SmallVectorImpl< Value * > &ReassocScalars)
	Peel associative single-use binary chains into operand columns.
static SmallVector< BoUpSLP::ValueList >	alignReassociatedOperandsByKey (ArrayRef< BoUpSLP::ValueList > Operands, function_ref< bool(unsigned Lane, unsigned Col)> IsNegated, const TargetLibraryInfo &TLI)
	Realign flattened operand columns so matching value families share a column across lanes.
static std::tuple< unsigned, unsigned, unsigned, int >	getReassocColumnsQuality (ArrayRef< BoUpSLP::ValueList > Columns, const BoUpSLP &R, unsigned Opcode)
	Rank reassociated operand layouts by vectorizable load columns, other load patterns, broadcast/constant columns, then fewer unique values per column.
static InstructionCost	canConvertToFMA (ArrayRef< Value * > VL, const InstructionsState &S, DominatorTree &DT, const DataLayout &DL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const TTI::TargetCostKind CostKind)
	Check if we can convert fadd/fsub sequence to FMAD.
static unsigned	getLoopTripCount (const Loop *L, ScalarEvolution &SE)
	Get the assumed loop trip count for the loop L.
static TTI::VectorInstrContext	getVectorInstrContextHint (ArrayRef< Value * > VL, const APInt &DemandedElts)
	Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.
static bool	isFirstInsertElement (const InsertElementInst *IE1, const InsertElementInst *IE2)
	Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.
template<typename T>
static T *	performExtractsShuffleAction (MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value *Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action)
	Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.
static Instruction *	propagateMetadata (Instruction *Inst, ArrayRef< Value * > VL)
static DebugLoc	getDebugLocFromPHI (PHINode &PN)
static bool	blockBodyHasVectorInstructions (BasicBlock *BB)
	Returns true if BB's body already contains vector instructions, e.g.
static RecurKind	getRdxKind (Value *V)
	Gets recurrence kind from the specified value.
static std::optional< unsigned >	getAggregateSize (Instruction *InsertInst)
static void	findBuildAggregateRec (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, unsigned OperandOffset, const BoUpSLP &R)
static bool	findBuildAggregate (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, const BoUpSLP &R)
	Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.
static Instruction *	getReductionInstr (const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI)
	Try and get a reduction instruction from a phi node.
static Instruction *	tryGetSecondaryReductionRoot (PHINode *Phi, Instruction *Root)
	We could have an initial reduction that is not an add.
template<typename T>
static bool	tryToVectorizeSequence (SmallVectorImpl< T * > &Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP &R)
template<bool IsCompatibility>
static bool	compareCmp (Value *V, Value *V2, TargetLibraryInfo &TLI, const DominatorTree &DT)
	Compare two cmp instructions.
static bool	isNonVectorizableInst (const Instruction *I, const TargetLibraryInfo *TLI)
	Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.
template<typename Func>
static void	forEachOperandChainCandidate (Instruction *I, Func F, bool ForReduction)
	Visits the value operands of I that are candidates for operand-chain vectorization.

Variables
static cl::opt< bool >	RunSLPVectorization ("vectorize-slp", cl::init(true), cl::Hidden, cl::desc("Run the SLP vectorization passes"))
static cl::opt< bool >	SLPReVec ("slp-revec", cl::init(false), cl::Hidden, cl::desc("Enable vectorization for wider vector utilization"))
static cl::opt< int >	SLPCostThreshold ("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this " "number "))
static cl::opt< bool >	ShouldVectorizeHor ("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions"))
static cl::opt< bool >	ShouldStartVectorizeHorAtStore ("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store"))
static cl::opt< bool >	SplitAlternateInstructions ("slp-split-alternate-instructions", cl::init(true), cl::Hidden, cl::desc("Improve the code quality by splitting alternate instructions"))
static cl::opt< bool >	SLPInstCountCheck ("slp-inst-count-check", cl::init(true), cl::Hidden, cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count"))
static cl::opt< int >	MaxVectorRegSizeOption ("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static cl::opt< unsigned >	MaxVFOption ("slp-max-vf", cl::init(0), cl::Hidden, cl::desc("Maximum SLP vectorization factor (0=unlimited)"))
static cl::opt< int >	ScheduleRegionSizeBudget ("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block"))
	Limits the size of scheduling regions in a block.
static cl::opt< int >	MinVectorRegSizeOption ("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static cl::opt< unsigned >	RecursionMaxDepth ("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree"))
static cl::opt< unsigned >	MinTreeSize ("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable"))
static cl::opt< unsigned >	PHINodeVectorizationBudget ("slp-phi-vectorization-budget", cl::init(1024), cl::Hidden, cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs"))
static cl::opt< int >	LookAheadMaxDepth ("slp-max-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for operand reordering scores"))
static cl::opt< int >	RootLookAheadMaxDepth ("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for searching best rooting option"))
static cl::opt< unsigned >	MinProfitableStridedLoads ("slp-min-strided-loads", cl::init(2), cl::Hidden, cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value"))
static cl::opt< unsigned >	MinProfitableStridedStores ("slp-min-strided-stores", cl::init(2), cl::Hidden, cl::desc("The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value"))
static cl::opt< unsigned >	MaxProfitableStride ("slp-max-stride", cl::init(8), cl::Hidden, cl::desc("The maximum stride, considered to be profitable."))
static cl::opt< bool >	EnableStridedStores ("slp-enable-strided-stores", cl::init(false), cl::Hidden, cl::desc("Enable SLP trees to be built from strided " "store chains."))
static cl::opt< bool >	EnableMaskedStores ("slp-enable-masked-stores", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores."))
static cl::opt< bool >	DisableTreeReorder ("slp-disable-tree-reorder", cl::init(false), cl::Hidden, cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only."))
static cl::opt< bool >	ForceStridedLoads ("slp-force-strided-loads", cl::init(false), cl::Hidden, cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only."))
static cl::opt< bool >	ViewSLPTree ("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz"))
static cl::opt< bool >	VectorizeNonPowerOf2 ("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, cl::desc("Try to vectorize with non-power-of-2 number of elements."))
static cl::opt< bool >	ForcePostProcessStoresOperands ("slp-postprocess-stores-operands", cl::init(false), cl::Hidden, cl::desc("Force vectorization of non-vectorizable stores operands."))
static cl::opt< bool >	NonVectReductions ("slp-non-vectorizables-as-reductions", cl::init(false), cl::Hidden, cl::desc("Use non-vectorizable instructions as potential reduction roots."))
static cl::opt< bool >	VectorizePoorThroughput ("slp-vectorize-poor-throughput", cl::init(true), cl::Hidden, cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds."))
static cl::opt< bool >	VectorizeOnceUsed ("slp-vectorize-once-used", cl::init(true), cl::Hidden, cl::desc("Use instructions with the single user as standalone " "vectorization seeds."))
static cl::opt< bool >	VectorizeCopyableElements ("slp-copyable-elements", cl::init(true), cl::Hidden, cl::desc("Try to replace values with the idempotent instructions for " "better vectorization."))
	Enables vectorization of copyable elements.
static cl::opt< bool >	VectorizeReassociatedOps ("slp-reassociate-ops", cl::init(true), cl::Hidden, cl::desc("Gather operands of associative binary chains into one node."))
	Gather operands of associative single-use binary chains into one node.
static cl::opt< unsigned >	ReassocReorderColumnLimit ("slp-reassociate-reorder-limit", cl::init(32), cl::Hidden, cl::desc("Max flattened operand columns for which associative-chain " "reordering runs the full operand reorder."))
	The family-realigned seed already groups the vectorizable columns; the VLOperands polish on top is quadratic in the column count, so past this many columns keep the seed instead.
static cl::opt< unsigned >	LoopAwareTripCount ("slp-cost-loop-trip-count", cl::init(2), cl::Hidden, cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)"))
static cl::opt< bool >	PerLaneGatherScale ("slp-per-lane-gather-scale", cl::init(true), cl::Hidden, cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences."))
	Refine the loop-aware cost scaling of gather/buildvector tree entries by using the per-lane execution scale of the operand that feeds each lane, instead of a single whole-entry scale.
static cl::opt< bool >	SLPEnableRuntimeAliasChecks ("slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden, cl::desc("Allow SLP to version a block with runtime alias checks to " "vectorize trees blocked by may-alias memory dependencies."))
	Enable versioning of a basic block with runtime alias checks.
static cl::opt< unsigned >	SLPMaxRuntimeAliasChecks ("slp-max-runtime-alias-checks", cl::init(8), cl::Hidden, cl::desc("The maximum number of runtime alias checks generated to guard a " "single SLP-vectorized region."))
	Maximum number of runtime alias checks (one per pair of base objects) that may guard a single versioned region.
static cl::opt< unsigned >	SLPRuntimeAliasChecksMaxScalarCostPercent ("slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25), cl::Hidden, cl::desc("Maximum SLP runtime alias check cost, as a percentage of the " "guarded scalar region cost, before versioning is rejected to " "avoid pessimizing the scalar fallback path."))
	The runtime checks and the guard branch execute on both the vector and the scalar fallback path, so they add overhead to the scalar code.
static const unsigned	AliasedCheckLimit = 10
static const unsigned	MaxMemDepDistance = 160
static const int	MinScheduleRegionSize = 16
	If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.
static const unsigned	MaxPHINumOperands = 128
	Maximum allowed number of operands in the PHI nodes.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "SLP"

Definition at line 124 of file SLPVectorizer.cpp.

SV_NAME

#define SV_NAME   "slp-vectorizer"

Definition at line 123 of file SLPVectorizer.cpp.

Function Documentation

alignReassociatedOperandsByKey()

SmallVector< BoUpSLP::ValueList > alignReassociatedOperandsByKey ( ArrayRef< BoUpSLP::ValueList > Operands, function_ref< bool(unsigned Lane, unsigned Col)> IsNegated, const TargetLibraryInfo & TLI )

static

Realign flattened operand columns so matching value families share a column across lanes.

Lane 0 is canonical; other lanes match by key equality only (never compare hash values). Columns sharing a key are paired by the family of their first operand where available, so e.g. shifts fed by the same load family land in one column instead of pairing by encounter order. Values move between columns only within the same sign: a subtracted leaf never lands in an added column. The sign is queried per lane and column with IsNegated: alternate add/sub nodes negate only the non-leading columns of their subtract lanes.

Definition at line 11732 of file SLPVectorizer.cpp.

References llvm::Aligned, llvm::dyn_cast(), E(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), generateKeySubkey(), llvm::getUnderlyingObject(), llvm::hash_value(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::map_to_vector(), Operands, P, llvm::seq(), and std::swap().

allStructUsersAreExtractValueInsts()

bool allStructUsersAreExtractValueInsts ( ArrayRef< Value * > VL )

static

Definition at line 9240 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::isa(), and llvm::IsaPred.

areTwoInsertFromSameBuildVector()

bool areTwoInsertFromSameBuildVector ( InsertElementInst * VU, InsertElementInst * V, function_ref< Value *(InsertElementInst *)> GetBaseOperand )

static

Check if two insertelement instructions are from the same buildvector.

Definition at line 6468 of file SLPVectorizer.cpp.

References llvm::dyn_cast_or_null(), llvm::VectorType::getElementCount(), llvm::slpvectorizer::getElementIndex(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::InsertElementInst::getType(), llvm::Value::hasOneUse(), llvm::SmallBitVector::set(), and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost(), and slpvectorizer::BoUpSLP::vectorizeTree().

blockBodyHasVectorInstructions()

bool blockBodyHasVectorInstructions ( BasicBlock * BB )

static

Returns true if BB's body already contains vector instructions, e.g.

from an earlier SLP vectorization in the same pass.

Definition at line 24171 of file SLPVectorizer.cpp.

References llvm::slpvectorizer::getValueType(), I, llvm::isa(), and SLPReVec.

Referenced by slpvectorizer::BoUpSLP::canVersionBlockForRuntimeChecks().

buildIntrinsicArgTypes()

SmallVector< Type * > buildIntrinsicArgTypes ( const CallInst * CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo * TTI )

static

Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.

Definition at line 8963 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), llvm::enumerate(), llvm::IntegerType::get(), llvm::Value::getContext(), llvm::slpvectorizer::getWidenedType(), llvm::isVectorIntrinsicWithScalarOpAtArg(), llvm::Intrinsic::not_intrinsic, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by getVectorOpCost().

canConvertToFMA()

InstructionCost canConvertToFMA ( ArrayRef< Value * > VL, const InstructionsState & S, DominatorTree & DT, const DataLayout & DL, TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const TTI::TargetCostKind CostKind )

static

Check if we can convert fadd/fsub sequence to FMAD.

Returns

Cost of the FMAD, if conversion is possible, invalid cost otherwise.

Definition at line 13360 of file SLPVectorizer.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, llvm::all_of(), llvm::FastMathFlags::allowContract(), Analysis, assert(), CostKind, DL, llvm::dyn_cast(), llvm::ArrayRef< T >::front(), llvm::slpvectorizer::InstructionsState::getAltOp(), llvm::slpvectorizer::InstructionsState::getAltOpcode(), llvm::InstructionCost::getInvalid(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::TargetTransformInfo::getOperandInfo(), llvm::slpvectorizer::getSameOpcode(), I, llvm::slpvectorizer::InstructionsState::isAddSubLikeOp(), llvm::slpvectorizer::InstructionsState::isAddSubOrFNegLikeOp(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::InstructionsState::isBitwiseLogicOp(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::slpvectorizer::InstructionsState::isMulDivLikeOp(), llvm::slpvectorizer::InstructionsState::isShiftOp(), Operands, llvm::FastMathFlags::set(), llvm::slpvectorizer::InstructionsState::valid(), and llvm::zip().

checkEVsForVecCalls()

bool checkEVsForVecCalls ( ArrayRef< Value * > VL, const InstructionsState & S, const TargetLibraryInfo & TLI, SmallVectorImpl< unsigned > & Indices, SmallVectorImpl< Value * > & Calls )

static

Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.

VL is a bundle whose state S is Instruction::ExtractValue. The bundle is acceptable for widening into one struct-of-vectors call only when:

• every element of VL is an ExtractValueInst,
• every ExtractValueInst extracts the same struct field (its getIndices() matches the main op's indices),
• the aggregate operands form a uniform set of CallInsts (per getSameOpcode) that is not an alt-shuffle and whose return type is a literal struct, and
• every user of every such call is itself an ExtractValueInst, so the external-use extraction code can rebuild scalars via extractvalue + extractelement without needing an insertvalue chain.

On success returns true and fills Indices with the common field index path and Calls with the per-lane aggregate calls (in VL order), for the caller to feed as the operand of the new tree entry. Otherwise returns false and leaves the output parameters untouched.

Definition at line 9196 of file SLPVectorizer.cpp.

References Aggregates, llvm::all_of(), llvm::slpvectorizer::allSameBlock(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::cast(), llvm::ArrayRef< T >::end(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::slpvectorizer::getSameOpcode(), llvm::Value::getType(), llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::IsaPred, IV, and llvm::SmallVectorImpl< T >::swap().

compareCmp()

template<bool IsCompatibility>

bool compareCmp ( Value * V, Value * V2, TargetLibraryInfo & TLI, const DominatorTree & DT )

static

Compare two cmp instructions.

If IsCompatibility is true, function returns true if 2 cmps have same/swapped predicates and mos compatible corresponding operands. If IsCompatibility is false, function implements strict weak ordering relation between two cmp instructions, returning true if the first instruction is "less" than the second, i.e. its predicate is less than the predicate of the second or the operands IDs are less than the operands IDs of the second cmp instruction.

Definition at line 33071 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), E(), llvm::DomTreeNodeBase< NodeT >::getDFSNumIn(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), llvm::User::getOperand(), llvm::slpvectorizer::getSameOpcode(), llvm::Type::getScalarSizeInBits(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::Type::getTypeID(), I, llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::isValidElementType(), and SLPReVec.

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts().

DEBUG_COUNTER()

DEBUG_COUNTER	(	VectorizedGraphs	,
		"slp-vectorized"	,
		"Controls which SLP graphs should be vectorized."	)

dumpOrder()

LLVM_DUMP_METHOD void dumpOrder ( const BoUpSLP::OrdersType & Order )

static

Definition at line 8180 of file SLPVectorizer.cpp.

References llvm::dbgs(), and LLVM_DUMP_METHOD.

findBuildAggregate()

bool findBuildAggregate ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, const BoUpSLP & R )

static

Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.

Also recognize homogeneous aggregates like {<2 x float>, <2 x float>}, {{float, float}, {float, float}}, [2 x {float, float}] and so on. See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples.

Assume LastInsertInst is of InsertElementInst or InsertValueInst type.

Returns

true if it matches.

Definition at line 32508 of file SLPVectorizer.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::erase(), findBuildAggregateRec(), getAggregateSize(), llvm::isa(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

findBuildAggregateRec()

void findBuildAggregateRec ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, unsigned OperandOffset, const BoUpSLP & R )

static

Definition at line 32469 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), findBuildAggregateRec(), getElementIndex(), llvm::User::getOperand(), llvm::Value::hasOneUse(), and llvm::isa().

Referenced by findBuildAggregate(), and findBuildAggregateRec().

findInnermostNonInvariantLoop()

const Loop * findInnermostNonInvariantLoop ( const Loop * L, ArrayRef< Value * > VL )

static

Find the innermost loop starting from L, for which at least a single value in VL is not invariant.

Definition at line 9145 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), and llvm::isa().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

forEachOperandChainCandidate()

template<typename Func>

void forEachOperandChainCandidate ( Instruction * I, Func F, bool ForReduction )

static

Visits the value operands of I that are candidates for operand-chain vectorization.

Definition at line 33265 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::enumerate(), F, I, llvm_unreachable, and NonVectReductions.

Referenced by llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

gatherPossiblyVectorizableLoads()

void gatherPossiblyVectorizableLoads ( const BoUpSLP & R, ArrayRef< Value * > VL, const DataLayout & DL, ScalarEvolution & SE, const TargetTransformInfo & TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > & GatheredLoads, bool AddNew = true )

static

Tries to find subvector of loads and builds new vector of only loads if can be profitable.

Definition at line 8243 of file SLPVectorizer.cpp.

References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::bit_ceil(), llvm::SetVector< T, Vector, Set, N >::contains(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::SmallSet< T, N, C >::contains(), llvm::Data, DL, llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::ArrayRef< T >::empty(), llvm::SetVector< T, Vector, Set, N >::empty(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::find_if(), llvm::ArrayRef< T >::front(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::LoadInst::getPointerOperand(), llvm::getPointersDiff(), llvm::Value::getType(), llvm::getUnderlyingObject(), llvm::slpvectorizer::getValueType(), llvm::has_single_bit(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SmallSet< T, N, C >::insert(), llvm::SetVector< T, Vector, Set, N >::insert_range(), llvm::slpvectorizer::isValidElementType(), llvm::Offset, P, RecursionMaxDepth, llvm::seq(), llvm::ArrayRef< T >::size(), SLPReVec, and llvm::zip().

generateKeySubkey()

std::pair< size_t, size_t > generateKeySubkey ( Value * V, const TargetLibraryInfo * TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate )

static

Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.

The keys/subkeys can be used for better sorting of the values themselves (keys) and in values subgroups (subkeys).

Definition at line 8867 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), Call, llvm::cast(), llvm::dyn_cast(), generateKeySubkey(), llvm::CmpInst::getInversePredicate(), llvm::VFDatabase::getMappings(), llvm::CmpInst::getSwappedPredicate(), llvm::getVectorIntrinsicIDForCall(), llvm::hash_combine(), llvm::hash_value(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::Instruction::isIntDivRem(), llvm::isTriviallyVectorizable(), llvm::slpvectorizer::isUndefVector(), llvm::slpvectorizer::isValidForAlternation(), and llvm::slpvectorizer::isVectorLikeInstWithConstOps().

Referenced by alignReassociatedOperandsByKey(), and generateKeySubkey().

getAggregateSize()

std::optional< unsigned > getAggregateSize ( Instruction * InsertInst )

static

Definition at line 32441 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::Type::isSingleValueType(), and IV.

Referenced by findBuildAggregate().

getDebugLocFromPHI()

DebugLoc getDebugLocFromPHI ( PHINode & PN )

static

Definition at line 22379 of file SLPVectorizer.cpp.

References DL, llvm::Instruction::getDebugLoc(), and llvm::DebugLoc::getUnknown().

getLoopTripCount()

unsigned getLoopTripCount ( const Loop * L, ScalarEvolution & SE )

static

Get the assumed loop trip count for the loop L.

Definition at line 15392 of file SLPVectorizer.cpp.

References llvm::getLoopEstimatedTripCount(), llvm::ScalarEvolution::getSmallConstantTripCount(), and LoopAwareTripCount.

getMainAltOpsNoStateVL()

std::pair< Instruction *, Instruction * > getMainAltOpsNoStateVL ( ArrayRef< Value * > VL )

static

Returns main/alternate instructions for the given VL.

Unlike getSameOpcode supports non-compatible instructions for better SplitVectorize node support.

Returns

first main/alt instructions, if only poisons and instruction with only 2 opcodes exists. Returns pair of nullptr otherwise.

Definition at line 9845 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), I, and llvm::isa().

getPoorThroughputTypeKey()

unsigned getPoorThroughputTypeKey ( Type * Ty )

static

Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.

fdiv float vs. fdiv double) is cached per width instead of sharing one verdict.

Definition at line 9052 of file SLPVectorizer.cpp.

Referenced by isPoorThroughputOp().

getRdxKind()

RecurKind getRdxKind ( Value * V )

static

Gets recurrence kind from the specified value.

Definition at line 32438 of file SLPVectorizer.cpp.

getReassocColumnsQuality()

std::tuple< unsigned, unsigned, unsigned, int > getReassocColumnsQuality ( ArrayRef< BoUpSLP::ValueList > Columns, const BoUpSLP & R, unsigned Opcode )

static

Rank reassociated operand layouts by vectorizable load columns, other load patterns, broadcast/constant columns, then fewer unique values per column.

Identity columns are scored by the opcode of the emitted combines.

Definition at line 11821 of file SLPVectorizer.cpp.

References llvm::all_of(), slpvectorizer::BoUpSLP::Gather, llvm::slpvectorizer::getReassocCombineOpcode(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::IsaPred, llvm::slpvectorizer::isBinOpIdentityConstant(), llvm::SmallPtrSetImplBase::size(), and slpvectorizer::BoUpSLP::Vectorize.

getReductionInstr()

Instruction * getReductionInstr ( const DominatorTree * DT, PHINode * P, BasicBlock * ParentBB, LoopInfo * LI )

static

Try and get a reduction instruction from a phi node.

Given a phi node P in a block ParentBB, consider possible reductions if they come from either ParentBB or a containing loop latch.

Returns

A candidate reduction value if possible, or

nullptr 

if not possible.

Definition at line 32543 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::isa(), and P.

getVectorCallCosts()

std::pair< InstructionCost, InstructionCost > getVectorCallCosts ( CallInst * CI, Type * VecTy, const TargetTransformInfo * TTI, const TargetLibraryInfo * TLI, ArrayRef< Type * > ArgTys, const TTI::TargetCostKind CostKind )

static

Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.

Returns invalid cost for the corresponding calls, if they cannot be vectorized/will be scalarized.

Definition at line 8988 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), CostKind, llvm::dyn_cast(), llvm::VFShape::get(), llvm::ElementCount::getFixed(), llvm::CallBase::getFunctionType(), llvm::InstructionCost::getInvalid(), llvm::slpvectorizer::getNumElements(), llvm::getVectorIntrinsicIDForCall(), llvm::VFDatabase::getVectorizedFunction(), IntrinsicCost, and llvm::CallBase::isNoBuiltin().

Referenced by getVectorOpCost().

getVectorInstrContextHint()

TTI::VectorInstrContext getVectorInstrContextHint ( ArrayRef< Value * > VL, const APInt & DemandedElts )

static

Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.

Definition at line 15691 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::TargetTransformInfo::getVectorInstrContextHint(), I, llvm::isa(), llvm::TargetTransformInfo::None, llvm::seq(), and llvm::ArrayRef< T >::size().

getVectorOpCost()

InstructionCost getVectorOpCost ( Instruction * I, unsigned VF, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const TTI::TargetCostKind CostKind )

static

Returns

the reciprocal-throughput cost of I widened to VF lanes (an arithmetic op or a vectorizable call).

Definition at line 9032 of file SLPVectorizer.cpp.

References assert(), buildIntrinsicArgTypes(), CostKind, llvm::dyn_cast(), getVectorCallCosts(), llvm::getVectorIntrinsicIDForCall(), llvm::slpvectorizer::getWidenedType(), I, and llvm::isa().

Referenced by isPoorThroughputOp().

isFirstInsertElement()

bool isFirstInsertElement ( const InsertElementInst * IE1, const InsertElementInst * IE2 )

static

Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.

Definition at line 18075 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), getElementIndex(), llvm::Value::hasOneUse(), and llvm_unreachable.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

isMainInstruction()

bool isMainInstruction ( Instruction * I, Instruction * MainOp, Instruction * AltOp, const TargetLibraryInfo & TLI )

static

Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.

Definition at line 13147 of file SLPVectorizer.cpp.

References llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), and I.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode().

isNonVectorizableInst()

bool isNonVectorizableInst ( const Instruction * I, const TargetLibraryInfo * TLI )

static

Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.

Definition at line 33212 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::any_of(), llvm::CallBase::args(), llvm::dyn_cast(), llvm::VFDatabase::getMappings(), getType(), llvm::getVectorIntrinsicIDForCall(), I, II, llvm::isa(), llvm::isTriviallyVectorizable(), and SLPReVec.

isPoorThroughputOp()

bool isPoorThroughputOp ( Instruction * I, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, PoorThroughputOpCache & Cache, const TTI::TargetCostKind CostKind )

static

Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.

fdiv, frem, fsqrt).

Definition at line 9097 of file SLPVectorizer.cpp.

References llvm::any_of(), CostKind, llvm::dyn_cast(), getPoorThroughputTypeKey(), llvm::getVectorIntrinsicIDForCall(), getVectorOpCost(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::slpvectorizer::isValidElementType(), SLPReVec, and llvm::TargetTransformInfo::TCC_Expensive.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

performExtractsShuffleAction()

template<typename T>

T * performExtractsShuffleAction ( MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value * Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action )

static

Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.

It tries to do it in several steps.

1. If the Base vector is not undef vector, resizing the very first mask to have common VF and perform action for 2 input vectors (including non-undef Base). Other shuffle masks are combined with the resulting after the 1 stage and processed as a shuffle of 2 elements.
2. If the Base is undef vector and have only 1 shuffle mask, perform the action only for 1 vector with the given mask, if it is not the identity mask.
3. If > 2 masks are used, perform the remaining shuffle actions for 2 vectors, combing the masks properly between the steps.

Definition at line 18130 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), assert(), llvm::sampleprof::Base, llvm::slpvectorizer::buildUseMask(), E(), I, llvm::slpvectorizer::isUndefVector(), llvm::slpvectorizer::isUndefVector< true >(), llvm::PoisonMaskElem, T, and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

propagateMetadata()

Instruction * propagateMetadata ( Instruction * Inst, ArrayRef< Value * > VL )

static

Returns

I after propagating metadata from VL only for instructions in VL.

Definition at line 22371 of file SLPVectorizer.cpp.

References llvm::isa(), llvm::propagateMetadata(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

scanAssociativeOperands()

void scanAssociativeOperands ( const InstructionsState & S, DominatorTree & DT, const DataLayout & DL, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const BoUpSLP & R, SmallVectorImpl< BoUpSLP::ValueList > & Operands, SmallBitVector & NegatedColumns, SmallVectorImpl< Value * > & ReassocScalars )

static

Peel associative single-use binary chains into operand columns.

Peeled instructions go to ReassocScalars; sub-operands are inserted after the peeled column so real pairs stay adjacent for flag reuse. Poison lanes are allowed through; a lane that is not itself a matching real instruction can still be peeled as a copyable identity leaf, as long as some other lane anchors the opcode. Add/sub (and fadd/fsub) chains peel together: a peeled subtract keeps the column sign for its first operand and flips it for the second one, recorded per column in NegatedColumns, so subtracted leaves stay out of the positive total.

Definition at line 11640 of file SLPVectorizer.cpp.

References llvm::all_of(), Analysis, llvm::slpvectorizer::InstructionsState::areInstructionsWithCopyableElements(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), DL, llvm::dyn_cast(), llvm::SmallBitVector::empty(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::slpvectorizer::getReassocCombineOpcode(), I, llvm::SmallVectorImpl< T >::insert(), llvm::slpvectorizer::InstructionsState::invalid(), llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::slpvectorizer::isReassocChainLink(), Operands, llvm::SmallBitVector::push_back(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

STATISTIC() [1/4]

STATISTIC	(	NumStoreChains	,
		"Number of vector stores created"	)

STATISTIC() [2/4]

STATISTIC	(	NumStridedStoreChains	,
		"Number of vectorized stride stores"	)

STATISTIC() [3/4]

STATISTIC	(	NumVectorInstructions	,
		"Number of vector instructions generated"	)

STATISTIC() [4/4]

STATISTIC	(	NumVectorizedStores	,
		"Number of vectorized stores"	)

tryGetSecondaryReductionRoot()

Instruction * tryGetSecondaryReductionRoot ( PHINode * Phi, Instruction * Root )

static

We could have an initial reduction that is not an add.

r *= v1 + v2 + v3 + v4 In such a case start looking for a tree rooted in the first '+'. \Returns the new root if found, which may be nullptr if not an instruction.

Definition at line 32591 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::User::getOperand(), llvm::isa(), LHS, and RHS.

tryToFindDuplicates()

bool tryToFindDuplicates ( SmallVectorImpl< Value * > & VL, SmallVectorImpl< int > & ReuseShuffleIndices, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const InstructionsState & S, const BoUpSLP::EdgeInfo & UserTreeIdx, const BoUpSLP & R, bool BuildGatherOnly = true )

static

Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.

The list of unique scalars is extended by poison values to the whole register size.

Returns

false if VL could not be uniquified, in which case VL is unchanged and ReuseShuffleIndices is empty.

Definition at line 9887 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::slpvectorizer::InstructionsState::areInstructionsWithCopyableElements(), assert(), slpvectorizer::BoUpSLP::BlendedLoadVectorize, llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::APInt::clearBit(), slpvectorizer::BoUpSLP::CompressVectorize, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), CostKind, llvm::CostThreshold, llvm::dbgs(), llvm::slpvectorizer::doesNotNeedToSchedule(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), llvm::SmallVectorImpl< T >::erase(), llvm::find_if_not(), llvm::SmallVectorTemplateCommon< T, typename >::front(), slpvectorizer::BoUpSLP::Gather, llvm::APInt::getAllOnes(), llvm::Instruction::getDataLayout(), llvm::slpvectorizer::getFullVectorNumberOfElements(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::getNumberOfParts(), llvm::User::getNumOperands(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::slpvectorizer::getScalarizationOverhead(), llvm::slpvectorizer::getShuffleCost(), llvm::DataLayout::getTypeSizeInBits(), llvm::slpvectorizer::getValueType(), llvm::slpvectorizer::getWidenedType(), llvm::APInt::getZero(), llvm::has_single_bit(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), InstructionCost, llvm::isa(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::IsaPred, isConstant(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::slpvectorizer::isVectorLikeInstWithConstOps(), LLVM_DEBUG, llvm::make_range(), N, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::rbegin(), llvm::SmallVectorTemplateCommon< T, typename >::rend(), slpvectorizer::BoUpSLP::ScatterVectorize, llvm::seq(), llvm::APInt::setBit(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, SLPReVec, slpvectorizer::BoUpSLP::StridedVectorize, llvm::TargetTransformInfo::TCC_Expensive, llvm::TargetTransformInfo::TCC_Free, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace(), and slpvectorizer::BoUpSLP::EdgeInfo::UserTE.

tryToVectorizeSequence()

template<typename T>

bool tryToVectorizeSequence ( SmallVectorImpl< T * > & Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP & R )

static

\Returns the minimum number of elements that we will attempt to vectorize.

Definition at line 32951 of file SLPVectorizer.cpp.

References llvm::ArrayRef(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), Changed, llvm::SmallVectorImpl< T >::clear(), llvm::dbgs(), llvm::dyn_cast(), E(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::slpvectorizer::getValueType(), I, llvm::isa(), llvm::slpvectorizer::isValidElementType(), LLVM_DEBUG, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), SLPReVec, llvm::stable_sort(), llvm::SmallVectorImpl< T >::swap(), and T.

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts(), and llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

Variable Documentation

AliasedCheckLimit

const unsigned AliasedCheckLimit = 10

static

Definition at line 337 of file SLPVectorizer.cpp.

DisableTreeReorder

cl::opt< bool > DisableTreeReorder("slp-disable-tree-reorder", cl::init(false), cl::Hidden, cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only.")) ( "slp-disable-tree-reorder" , cl::init(false) , cl::Hidden , cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only.")  )

static

Referenced by slpvectorizer::BoUpSLP::isProfitableToReorder().

EnableMaskedStores

cl::opt< bool > EnableMaskedStores("slp-enable-masked-stores", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores.")) ( "slp-enable-masked-stores" , cl::init(true) , cl::Hidden , cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores.")  )

static

EnableStridedStores

cl::opt< bool > EnableStridedStores("slp-enable-strided-stores", cl::init(false), cl::Hidden, cl::desc("Enable SLP trees to be built from strided " "store chains.")) ( "slp-enable-strided-stores" , cl::init(false) , cl::Hidden , cl::desc("Enable SLP trees to be built from strided " "store chains.")  )

static

ForcePostProcessStoresOperands

cl::opt< bool > ForcePostProcessStoresOperands("slp-postprocess-stores-operands", cl::init(false), cl::Hidden, cl::desc("Force vectorization of non-vectorizable stores operands.")) ( "slp-postprocess-stores-operands" , cl::init(false) , cl::Hidden , cl::desc("Force vectorization of non-vectorizable stores operands.")  )

static

ForceStridedLoads

cl::opt< bool > ForceStridedLoads("slp-force-strided-loads", cl::init(false), cl::Hidden, cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only.")) ( "slp-force-strided-loads" , cl::init(false) , cl::Hidden , cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only.")  )

static

LookAheadMaxDepth

cl::opt< int > LookAheadMaxDepth("slp-max-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for operand reordering scores")) ( "slp-max-look-ahead-depth" , cl::init(2) , cl::Hidden , cl::desc("The maximum look-ahead depth for operand reordering scores")  )

static

LoopAwareTripCount

cl::opt< unsigned > LoopAwareTripCount("slp-cost-loop-trip-count", cl::init(2), cl::Hidden, cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)")) ( "slp-cost-loop-trip-count" , cl::init(2) , cl::Hidden , cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)")  )

static

Referenced by getLoopTripCount(), and slpvectorizer::BoUpSLP::getTreeCost().

MaxMemDepDistance

const unsigned MaxMemDepDistance = 160

static

Definition at line 342 of file SLPVectorizer.cpp.

Referenced by slpvectorizer::BoUpSLP::canVersionForRuntimeChecks().

MaxPHINumOperands

const unsigned MaxPHINumOperands = 128

static

Maximum allowed number of operands in the PHI nodes.

Definition at line 349 of file SLPVectorizer.cpp.

MaxProfitableStride

cl::opt< unsigned > MaxProfitableStride("slp-max-stride", cl::init(8), cl::Hidden, cl::desc("The maximum stride, considered to be profitable.")) ( "slp-max-stride" , cl::init(8) , cl::Hidden , cl::desc("The maximum stride, considered to be profitable.")  )

static

Referenced by slpvectorizer::BoUpSLP::isStridedLoad().

MaxVectorRegSizeOption

cl::opt< int > MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) ( "slp-max-reg-size" , cl::init(128) , cl::Hidden , cl::desc("Attempt to vectorize for this register size in bits")  )

static

Referenced by slpvectorizer::BoUpSLP::BoUpSLP().

MaxVFOption

cl::opt< unsigned > MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden, cl::desc("Maximum SLP vectorization factor (0=unlimited)")) ( "slp-max-vf" , cl::init(0) , cl::Hidden , cl::desc("Maximum SLP vectorization factor (0=unlimited)")  )

static

Referenced by slpvectorizer::BoUpSLP::getMaximumVF().

MinProfitableStridedLoads

cl::opt< unsigned > MinProfitableStridedLoads("slp-min-strided-loads", cl::init(2), cl::Hidden, cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value")) ( "slp-min-strided-loads" , cl::init(2) , cl::Hidden , cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value")  )

static

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate(), and slpvectorizer::BoUpSLP::isStridedLoad().

MinProfitableStridedStores

cl::opt< unsigned > MinProfitableStridedStores("slp-min-strided-stores", cl::init(2), cl::Hidden, cl::desc( "The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value")) ( "slp-min-strided-stores" , cl::init(2) , cl::Hidden , cl::desc( "The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value")  )

static

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate().

MinScheduleRegionSize

const int MinScheduleRegionSize = 16

static

If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

Definition at line 346 of file SLPVectorizer.cpp.

MinTreeSize

cl::opt< unsigned > MinTreeSize("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable")) ( "slp-min-tree-size" , cl::init(3) , cl::Hidden , cl::desc("Only vectorize small trees if they are fully vectorizable")  )

static

Referenced by slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

MinVectorRegSizeOption

cl::opt< int > MinVectorRegSizeOption("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) ( "slp-min-reg-size" , cl::init(128) , cl::Hidden , cl::desc("Attempt to vectorize for this register size in bits")  )

static

Referenced by slpvectorizer::BoUpSLP::BoUpSLP().

NonVectReductions

cl::opt< bool > NonVectReductions("slp-non-vectorizables-as-reductions", cl::init(false), cl::Hidden, cl::desc( "Use non-vectorizable instructions as potential reduction roots.")) ( "slp-non-vectorizables-as-reductions" , cl::init(false) , cl::Hidden , cl::desc( "Use non-vectorizable instructions as potential reduction roots.")  )

static

Referenced by forEachOperandChainCandidate().

PerLaneGatherScale

cl::opt< bool > PerLaneGatherScale("slp-per-lane-gather-scale", cl::init(true), cl::Hidden, cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences.")) ( "slp-per-lane-gather-scale" , cl::init(true) , cl::Hidden , cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences.")  )

static

Refine the loop-aware cost scaling of gather/buildvector tree entries by using the per-lane execution scale of the operand that feeds each lane, instead of a single whole-entry scale.

This matches the LICM hoisting performed by optimizeGatherSequence() at codegen time: lanes whose operands are loop-invariant in an inner loop contribute the outer loop's execution scale rather than the inner loop's, which avoids over-costing buildvectors that bridge values from outer loop nests into an inner loop.

PHINodeVectorizationBudget

cl::opt< unsigned > PHINodeVectorizationBudget("slp-phi-vectorization-budget", cl::init(1024), cl::Hidden, cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs")) ( "slp-phi-vectorization-budget" , cl::init(1024) , cl::Hidden , cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs")  )

static

ReassocReorderColumnLimit

cl::opt< unsigned > ReassocReorderColumnLimit("slp-reassociate-reorder-limit", cl::init(32), cl::Hidden, cl::desc("Max flattened operand columns for which associative-chain " "reordering runs the full operand reorder.")) ( "slp-reassociate-reorder-limit" , cl::init(32) , cl::Hidden , cl::desc("Max flattened operand columns for which associative-chain " "reordering runs the full operand reorder.")  )

static

The family-realigned seed already groups the vectorizable columns; the VLOperands polish on top is quadratic in the column count, so past this many columns keep the seed instead.

RecursionMaxDepth

cl::opt< unsigned > RecursionMaxDepth("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree")) ( "slp-recursion-max-depth" , cl::init(12) , cl::Hidden , cl::desc("Limit the recursion depth when building a vectorizable tree")  )

static

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), slpvectorizer::BoUpSLP::findPartiallyOrderedLoads(), gatherPossiblyVectorizableLoads(), slpvectorizer::BoUpSLP::getVectorElementSize(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

RootLookAheadMaxDepth

cl::opt< int > RootLookAheadMaxDepth("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for searching best rooting option")) ( "slp-max-root-look-ahead-depth" , cl::init(2) , cl::Hidden , cl::desc("The maximum look-ahead depth for searching best rooting option")  )

static

Referenced by slpvectorizer::BoUpSLP::findBestRootPair().

RunSLPVectorization

cl::opt< bool > RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden, cl::desc("Run the SLP vectorization passes")) ( "vectorize-slp" , cl::init(true) , cl::Hidden , cl::desc("Run the SLP vectorization passes")  )

static

Referenced by llvm::SLPVectorizerPass::runImpl().

ScheduleRegionSizeBudget

cl::opt< int > ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block")) ( "slp-schedule-budget" , cl::init(100000) , cl::Hidden , cl::desc("Limit the size of the SLP scheduling region per block")  )

static

Limits the size of scheduling regions in a block.

It avoid long compile times for very large blocks where vector instructions are spread over a wide range. This limit is way higher than needed by real-world functions.

Referenced by slpvectorizer::BoUpSLP::getSpillCost().

ShouldStartVectorizeHorAtStore

cl::opt< bool > ShouldStartVectorizeHorAtStore("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc( "Attempt to vectorize horizontal reductions feeding into a store")) ( "slp-vectorize-hor-store" , cl::init(false) , cl::Hidden , cl::desc( "Attempt to vectorize horizontal reductions feeding into a store")  )

static

ShouldVectorizeHor

cl::opt< bool > ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions")) ( "slp-vectorize-hor" , cl::init(true) , cl::Hidden , cl::desc("Attempt to vectorize horizontal reductions")  )

static

SLPCostThreshold

cl::opt< int > SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this " "number ")) ( "slp-threshold" , cl::init(0) , cl::Hidden , cl::desc("Only vectorize if you gain more than this " "number ")  )

static

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), slpvectorizer::BoUpSLP::getTreeCost(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

SLPEnableRuntimeAliasChecks

cl::opt< bool > SLPEnableRuntimeAliasChecks("slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden, cl::desc("Allow SLP to version a block with runtime alias checks to " "vectorize trees blocked by may-alias memory dependencies.")) ( "slp-vectorize-with-runtime-alias-checks" , cl::init(true) , cl::Hidden , cl::desc("Allow SLP to version a block with runtime alias checks to " "vectorize trees blocked by may-alias memory dependencies.")  )

static

Enable versioning of a basic block with runtime alias checks.

SLPInstCountCheck

cl::opt< bool > SLPInstCountCheck("slp-inst-count-check", cl::init(true), cl::Hidden, cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count")) ( "slp-inst-count-check" , cl::init(true) , cl::Hidden , cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

SLPMaxRuntimeAliasChecks

cl::opt< unsigned > SLPMaxRuntimeAliasChecks("slp-max-runtime-alias-checks", cl::init(8), cl::Hidden, cl::desc("The maximum number of runtime alias checks generated to guard a " "single SLP-vectorized region.")) ( "slp-max-runtime-alias-checks" , cl::init(8) , cl::Hidden , cl::desc("The maximum number of runtime alias checks generated to guard a " "single SLP-vectorized region.")  )

static

Maximum number of runtime alias checks (one per pair of base objects) that may guard a single versioned region.

Referenced by slpvectorizer::BoUpSLP::canVersionForRuntimeChecks().

SLPReVec

cl::opt< bool > SLPReVec("slp-revec", cl::init(false), cl::Hidden, cl::desc("Enable vectorization for wider vector utilization")) ( "slp-revec" , cl::init(false) , cl::Hidden , cl::desc("Enable vectorization for wider vector utilization")  )

static

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), blockBodyHasVectorInstructions(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canMapToVector(), slpvectorizer::BoUpSLP::canVectorizeLoads(), compareCmp(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather(), gatherPossiblyVectorizableLoads(), slpvectorizer::BoUpSLP::getInsertBuildVectorSrcTy(), slpvectorizer::BoUpSLP::getNumberOfParts(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getTreeCost(), isNonVectorizableInst(), isPoorThroughputOp(), slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable(), slpvectorizer::BoUpSLP::optimizeGatherSequence(), slpvectorizer::BoUpSLP::reorderBottomToTop(), slpvectorizer::BoUpSLP::reorderTopToBottom(), tryToFindDuplicates(), tryToVectorizeSequence(), llvm::SLPVectorizerPass::vectorizeCmpInsts(), llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts(), and slpvectorizer::BoUpSLP::vectorizeTree().

SLPRuntimeAliasChecksMaxScalarCostPercent

cl::opt< unsigned > SLPRuntimeAliasChecksMaxScalarCostPercent("slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25), cl::Hidden, cl::desc("Maximum SLP runtime alias check cost, as a percentage of the " "guarded scalar region cost, before versioning is rejected to " "avoid pessimizing the scalar fallback path.")) ( "slp-runtime-alias-checks-max-scalar-cost-percent" , cl::init(25) , cl::Hidden , cl::desc("Maximum SLP runtime alias check cost, as a percentage of the " "guarded scalar region cost, before versioning is rejected to " "avoid pessimizing the scalar fallback path.")  )

static

The runtime checks and the guard branch execute on both the vector and the scalar fallback path, so they add overhead to the scalar code.

Referenced by slpvectorizer::BoUpSLP::canVersionForRuntimeChecks().

SplitAlternateInstructions

cl::opt< bool > SplitAlternateInstructions("slp-split-alternate-instructions", cl::init(true), cl::Hidden, cl::desc("Improve the code quality by splitting alternate instructions")) ( "slp-split-alternate-instructions" , cl::init(true) , cl::Hidden , cl::desc("Improve the code quality by splitting alternate instructions")  )

static

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode().

VectorizeCopyableElements

cl::opt< bool > VectorizeCopyableElements("slp-copyable-elements", cl::init(true), cl::Hidden, cl::desc("Try to replace values with the idempotent instructions for " "better vectorization.")) ( "slp-copyable-elements" , cl::init(true) , cl::Hidden , cl::desc("Try to replace values with the idempotent instructions for " "better vectorization.")  )

static

Enables vectorization of copyable elements.

VectorizeNonPowerOf2

cl::opt< bool > VectorizeNonPowerOf2("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, cl::desc("Try to vectorize with non-power-of-2 number of elements.")) ( "slp-vectorize-non-power-of-2" , cl::init(false) , cl::Hidden , cl::desc("Try to vectorize with non-power-of-2 number of elements.")  )

static

VectorizeOnceUsed

cl::opt< bool > VectorizeOnceUsed("slp-vectorize-once-used", cl::init(true), cl::Hidden, cl::desc("Use instructions with the single user as standalone " "vectorization seeds.")) ( "slp-vectorize-once-used" , cl::init(true) , cl::Hidden , cl::desc("Use instructions with the single user as standalone " "vectorization seeds.")  )

static

Referenced by llvm::SLPVectorizerPass::runImpl().

VectorizePoorThroughput

cl::opt< bool > VectorizePoorThroughput("slp-vectorize-poor-throughput", cl::init(true), cl::Hidden, cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds.")) ( "slp-vectorize-poor-throughput" , cl::init(true) , cl::Hidden , cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds.")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

VectorizeReassociatedOps

cl::opt< bool > VectorizeReassociatedOps("slp-reassociate-ops", cl::init(true), cl::Hidden, cl::desc("Gather operands of associative binary chains into one node.")) ( "slp-reassociate-ops" , cl::init(true) , cl::Hidden , cl::desc("Gather operands of associative binary chains into one node.")  )

static

Gather operands of associative single-use binary chains into one node.

ViewSLPTree

cl::opt< bool > ViewSLPTree("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz")) ( "view-slp-tree" , cl::Hidden , cl::desc("Display the SLP trees with Graphviz")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().