SLPVectorizer.cpp File Reference

#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "SLPVectorizer/SLPUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	slpvectorizer::BoUpSLP
	Bottom Up SLP Vectorizer. More...
struct	slpvectorizer::BoUpSLP::StridedPtrInfo
	If we decide to generate strided load / store, this struct contains all the necessary info. More...
struct	slpvectorizer::BoUpSLP::EdgeInfo
	This structure holds any data we need about the edges being traversed during buildTreeRec(). More...
class	slpvectorizer::BoUpSLP::LookAheadHeuristics
	A helper class used for scoring candidates for two consecutive lanes. More...
class	slpvectorizer::BoUpSLP::VLOperands
	A helper data structure to hold the operands of a vector of instructions. More...
struct	llvm::DenseMapInfo< BoUpSLP::EdgeInfo >
struct	llvm::GraphTraits< BoUpSLP * >
struct	llvm::GraphTraits< BoUpSLP * >::ChildIteratorType
	Add the VectorizableTree to the index iterator to be able to return TreeEntry pointers. More...
class	llvm::GraphTraits< BoUpSLP * >::nodes_iterator
	For the node iterator we just need to turn the TreeEntry iterator into a TreeEntry* iterator so that it dereferences to NodeRef. More...
struct	llvm::DOTGraphTraits< BoUpSLP * >
class	slpvectorizer::BoUpSLP::ShuffleCostEstimator
	Merges shuffle masks and emits final shuffle instruction, if required. More...
class	slpvectorizer::BoUpSLP::ShuffleInstructionBuilder
	Merges shuffle masks and emits final shuffle instruction, if required. More...

Macros
#define	SV_NAME   "slp-vectorizer"
#define	DEBUG_TYPE   "SLP"

Functions
	STATISTIC (NumVectorInstructions, "Number of vector instructions generated")
	STATISTIC (NumStridedStoreChains, "Number of vectorized stride stores")
	STATISTIC (NumStoreChains, "Number of vector stores created")
	STATISTIC (NumVectorizedStores, "Number of vectorized stores")
	DEBUG_COUNTER (VectorizedGraphs, "slp-vectorized", "Controls which SLP graphs should be vectorized.")
static bool	isAllowedNonPowerOf2VF (unsigned NumElts)
	True when slp-vectorize-non-power-of-2 is enabled and NumElts is a supported non-power-of-2 width: NumElts + 1 must be a power of two (e.g.
static bool	isValidElementType (Type *Ty)
	Predicate for the element types that the SLP vectorizer supports.
static Type *	getValueType (Value *V, bool LookThroughCmp=false)
	Returns the "element type" of the given value/instruction V.
static Type *	getWidenedType (Type *ScalarTy, unsigned VF)
static unsigned	getFullVectorNumberOfElements (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz)
	Returns the number of elements of the given type Ty, not less than Sz, which forms type, which splits by TTI into whole vector types during legalization.
static unsigned	getFloorFullVectorNumberOfElements (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz)
	Returns the number of elements of the given type Ty, not greater than Sz, which forms type, which splits by TTI into whole vector types during legalization.
static void	transformScalarShuffleIndiciesToVector (unsigned VecTyNumElements, SmallVectorImpl< int > &Mask)
static unsigned	getShufflevectorNumGroups (ArrayRef< Value * > VL)
static SmallVector< int >	calculateShufflevectorMask (ArrayRef< Value * > VL)
static std::string	shortBundleName (ArrayRef< Value * > VL, int Idx=-1)
	Print a short descriptor of the instruction bundle suitable for debug output.
static bool	isCommutative (const Instruction *I, const Value *ValWithUses, bool IsCopyable=false)
static bool	isCommutableOperand (const Instruction *I, Value *ValWithUses, unsigned Op, bool IsCopyable=false)
	Checks if the operand is commutative.
static bool	isCommutative (const Instruction *I)
	This is a helper function to check whether I is commutative.
static unsigned	getNumberOfPotentiallyCommutativeOps (Instruction *I)
static std::optional< unsigned >	getElementIndex (const Value *Inst, unsigned Offset=0)
static SmallBitVector	buildUseMask (int VF, ArrayRef< int > Mask, UseMask MaskArg)
	Prepares a use bitset for the given mask either for the first argument or for the second.
template<bool IsPoisonOnly = false>
static SmallBitVector	isUndefVector (const Value *V, const SmallBitVector &UseMask={})
	Checks if the given value is actually an undefined constant vector.
static std::optional< TargetTransformInfo::ShuffleKind >	isFixedVectorShuffle (ArrayRef< Value * > VL, SmallVectorImpl< int > &Mask, AssumptionCache *AC)
	Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.
static bool	areAllOperandsNonInsts (Value *V)
	Checks if the provided value does not require scheduling.
static bool	isUsedOutsideBlock (Value *V)
	Checks if the provided value does not require scheduling.
static bool	doesNotNeedToBeScheduled (Value *V)
	Checks if the specified value does not require scheduling.
static bool	isValidForAlternation (unsigned Opcode)
static InstructionsState	getSameOpcode (ArrayRef< Value * > VL, const TargetLibraryInfo &TLI)
static Instruction *	findInstructionWithOpcode (ArrayRef< Value * > VL, unsigned Opcode)
	Find an instruction with a specific opcode in VL.
static bool	areCompatibleCmpOps (Value *BaseOp0, Value *BaseOp1, Value *Op0, Value *Op1, const TargetLibraryInfo &TLI)
	Checks if the provided operands of 2 cmp instructions are compatible, i.e.
static bool	isCmpSameOrSwapped (const CmpInst *BaseCI, const CmpInst *CI, const TargetLibraryInfo &TLI)
static bool	doesInTreeUserNeedToExtract (Value *Scalar, Instruction *UserInst, TargetLibraryInfo *TLI, const TargetTransformInfo *TTI)
static MemoryLocation	getLocation (Instruction *I)
static bool	isSimple (Instruction *I)
static void	addMask (SmallVectorImpl< int > &Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs=false)
	Shuffles Mask in accordance with the given SubMask.
static void	fixupOrderingIndices (MutableArrayRef< unsigned > Order)
	Order may have elements assigned special value (size) which is out of bounds.
static SmallBitVector	getAltInstrMask (ArrayRef< Value * > VL, Type *ScalarTy, unsigned Opcode0, unsigned Opcode1)
static SmallVector< Constant * >	replicateMask (ArrayRef< Constant * > Val, unsigned VF)
	Replicates the given Val VF times.
static void	inversePermutation (ArrayRef< unsigned > Indices, SmallVectorImpl< int > &Mask)
static void	reorderScalars (SmallVectorImpl< Value * > &Scalars, ArrayRef< int > Mask)
	Reorders the list of scalars in accordance with the given Mask.
static bool	doesNotNeedToSchedule (ArrayRef< Value * > VL)
	Checks if the specified array of instructions does not require scheduling.
static bool	hasFullVectorsOrPowerOf2 (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz)
	Returns true if widened type of Ty elements with size Sz represents full vector type, i.e.
static unsigned	getNumberOfParts (const TargetTransformInfo &TTI, Type *VecTy, Type *ScalarTy, const unsigned Limit=std::numeric_limits< unsigned >::max())
	Returns number of parts, the type VecTy will be split at the codegen phase.
static void	reorderReuses (SmallVectorImpl< int > &Reuses, ArrayRef< int > Mask)
	Reorders the given Reuses mask according to the given Mask.
static void	reorderOrder (SmallVectorImpl< unsigned > &Order, ArrayRef< int > Mask, bool BottomOrder=false)
	Reorders the given Order according to the given Mask.
static bool	arePointersCompatible (Value *Ptr1, Value *Ptr2, const TargetLibraryInfo &TLI, bool CompareOpcodes=true)
template<typename T>
static Align	computeCommonAlignment (ArrayRef< Value * > VL)
	Calculates minimal alignment as a common alignment.
static bool	isReverseOrder (ArrayRef< unsigned > Order)
	Check if Order represents reverse order.
static const SCEV *	calculateRtStride (ArrayRef< Value * > PointerOps, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
	Checks if the provided list of pointers Pointers represents the strided pointers for type ElemTy.
static std::pair< InstructionCost, InstructionCost >	getGEPCosts (const TargetTransformInfo &TTI, ArrayRef< Value * > Ptrs, Value *BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind, Type *ScalarTy, VectorType *VecTy)
	Calculate the scalar and the vector costs from vectorizing set of GEPs.
static InstructionCost	getShuffleCost (const TargetTransformInfo &TTI, TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask={}, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={})
	Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.
static InstructionCost	getScalarizationOverhead (const TargetTransformInfo &TTI, Type *ScalarTy, VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None)
	This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorType, a vector will be inserted or extracted instead of a scalar.
static InstructionCost	getVectorInstrCost (const TargetTransformInfo &TTI, Type *ScalarTy, unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar, ArrayRef< std::tuple< Value *, User *, int > > ScalarUserAndIdx)
	This is similar to TargetTransformInfo::getVectorInstrCost, but if ScalarTy is a FixedVectorType, a vector will be extracted instead of a scalar.
static InstructionCost	getExtractWithExtendCost (const TargetTransformInfo &TTI, unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput)
	This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst is a FixedVectorType, a vector will be extracted instead of a scalar.
static Value *	createInsertVector (IRBuilderBase &Builder, Value *Vec, Value *V, unsigned Index, function_ref< Value *(Value *, Value *, ArrayRef< int >)> Generator={})
	Creates subvector insert.
static Value *	createExtractVector (IRBuilderBase &Builder, Value *Vec, unsigned SubVecVF, unsigned Index)
	Generates subvector extract using Generator or using default shuffle.
static bool	buildCompressMask (ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, Type *ScalarTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< int > &CompressMask)
	Builds compress-like mask for shuffles for the given PointerOps, ordered with Order.
static bool	isMaskedLoadCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, AssumptionCache &AC, const DominatorTree &DT, const TargetLibraryInfo &TLI, const function_ref< bool(Value *)> AreAllUsersVectorized, bool &IsMasked, unsigned &InterleaveFactor, SmallVectorImpl< int > &CompressMask, VectorType *&LoadVecTy)
	Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.
static bool	isMaskedLoadCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, AssumptionCache &AC, const DominatorTree &DT, const TargetLibraryInfo &TLI, const function_ref< bool(Value *)> AreAllUsersVectorized)
	Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.
static bool	isMaskedStoreCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, Align CommonAlignment, SmallVectorImpl< int > &ReuseShuffleIndices, FixedVectorType *&StoreVecTy)
	Checks if the stores VL with pointers PointerOps can be lowered as a single masked store.
static bool	clusterSortPtrAccesses (ArrayRef< Value * > VL, ArrayRef< BasicBlock * > BBs, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
static bool	areTwoInsertFromSameBuildVector (InsertElementInst *VU, InsertElementInst *V, function_ref< Value *(InsertElementInst *)> GetBaseOperand)
	Check if two insertelement instructions are from the same buildvector.
static bool	isAlternateInstruction (Instruction *I, Instruction *MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction I is an alternate operation for the given MainOp and AltOp instructions.
static bool	isRepeatedNonIdentityClusteredMask (ArrayRef< int > Mask, unsigned Sz)
	Checks if the given mask is a "clustered" mask with the same clusters of size Sz, which are not identity submasks.
static void	combineOrders (MutableArrayRef< unsigned > Order, ArrayRef< unsigned > SecondaryOrder)
static LLVM_DUMP_METHOD void	dumpOrder (const BoUpSLP::OrdersType &Order)
static void	gatherPossiblyVectorizableLoads (const BoUpSLP &R, ArrayRef< Value * > VL, const DataLayout &DL, ScalarEvolution &SE, const TargetTransformInfo &TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > &GatheredLoads, bool AddNew=true)
	Tries to find subvector of loads and builds new vector of only loads if can be profitable.
static std::pair< size_t, size_t >	generateKeySubkey (Value *V, const TargetLibraryInfo *TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate)
	Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.
static bool	isMainInstruction (Instruction *I, Instruction *MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.
static SmallVector< Type * >	buildIntrinsicArgTypes (const CallInst *CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo *TTI)
	Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.
static std::pair< InstructionCost, InstructionCost >	getVectorCallCosts (CallInst *CI, Type *VecTy, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, ArrayRef< Type * > ArgTys)
	Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.
static InstructionCost	getVectorOpCost (Instruction *I, unsigned VF, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI)
static unsigned	getPoorThroughputTypeKey (Type *Ty)
	Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.
static bool	isPoorThroughputOp (Instruction *I, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, PoorThroughputOpCache &Cache)
	Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.
static const Loop *	findInnermostNonInvariantLoop (const Loop *L, ArrayRef< Value * > VL)
	Find the innermost loop starting from L, for which at least a single value in VL is not invariant.
static bool	checkEVsForVecCalls (ArrayRef< Value * > VL, const InstructionsState &S, const TargetLibraryInfo &TLI, SmallVectorImpl< unsigned > &Indices, SmallVectorImpl< Value * > &Calls)
	Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.
static bool	allStructUsersAreExtractValueInsts (ArrayRef< Value * > VL)
static std::pair< Instruction *, Instruction * >	getMainAltOpsNoStateVL (ArrayRef< Value * > VL)
	Returns main/alternate instructions for the given VL.
static bool	tryToFindDuplicates (SmallVectorImpl< Value * > &VL, SmallVectorImpl< int > &ReuseShuffleIndices, const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const InstructionsState &S, const BoUpSLP::EdgeInfo &UserTreeIdx, const BoUpSLP &R, bool BuildGatherOnly=true)
	Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.
static InstructionCost	canConvertToFMA (ArrayRef< Value * > VL, const InstructionsState &S, DominatorTree &DT, const DataLayout &DL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI)
	Check if we can convert fadd/fsub sequence to FMAD.
static unsigned	getLoopTripCount (const Loop *L, ScalarEvolution &SE)
	Get the assumed loop trip count for the loop L.
static TTI::VectorInstrContext	getVectorInstrContextHint (ArrayRef< Value * > VL, const APInt &DemandedElts)
	Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.
static bool	isFirstInsertElement (const InsertElementInst *IE1, const InsertElementInst *IE2)
	Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.
template<typename T>
static T *	performExtractsShuffleAction (MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value *Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action)
	Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.
static Instruction *	propagateMetadata (Instruction *Inst, ArrayRef< Value * > VL)
static DebugLoc	getDebugLocFromPHI (PHINode &PN)
static RecurKind	getRdxKind (Value *V)
	Gets recurrence kind from the specified value.
static std::optional< unsigned >	getAggregateSize (Instruction *InsertInst)
static void	findBuildAggregateRec (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, unsigned OperandOffset, const BoUpSLP &R)
static bool	findBuildAggregate (Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, const BoUpSLP &R)
	Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.
static Instruction *	getReductionInstr (const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI)
	Try and get a reduction instruction from a phi node.
static bool	matchRdxBop (Instruction *I, Value *&V0, Value *&V1)
static Instruction *	tryGetSecondaryReductionRoot (PHINode *Phi, Instruction *Root)
	We could have an initial reduction that is not an add.
static Instruction *	getNonPhiOperand (Instruction *I, PHINode *Phi)
	Returns the first operand of I that does not match Phi.
static bool	isReductionCandidate (Instruction *I)
	\Returns true if I is a candidate instruction for reduction vectorization.
template<typename T>
static bool	tryToVectorizeSequence (SmallVectorImpl< T * > &Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP &R)
template<bool IsCompatibility>
static bool	compareCmp (Value *V, Value *V2, TargetLibraryInfo &TLI, const DominatorTree &DT)
	Compare two cmp instructions.
static bool	isNonVectorizableInst (const Instruction *I, const TargetLibraryInfo *TLI)
	Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.
template<typename Func>
static void	forEachOperandChainCandidate (Instruction *I, Func F, bool ForReduction)
	Visits the value operands of I that are candidates for operand-chain vectorization.

Variables
static cl::opt< bool >	RunSLPVectorization ("vectorize-slp", cl::init(true), cl::Hidden, cl::desc("Run the SLP vectorization passes"))
static cl::opt< bool >	SLPReVec ("slp-revec", cl::init(false), cl::Hidden, cl::desc("Enable vectorization for wider vector utilization"))
static cl::opt< int >	SLPCostThreshold ("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this " "number "))
static cl::opt< bool >	ShouldVectorizeHor ("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions"))
static cl::opt< bool >	ShouldStartVectorizeHorAtStore ("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store"))
static cl::opt< bool >	SplitAlternateInstructions ("slp-split-alternate-instructions", cl::init(true), cl::Hidden, cl::desc("Improve the code quality by splitting alternate instructions"))
static cl::opt< bool >	SLPInstCountCheck ("slp-inst-count-check", cl::init(true), cl::Hidden, cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count"))
static cl::opt< int >	MaxVectorRegSizeOption ("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static cl::opt< unsigned >	MaxVFOption ("slp-max-vf", cl::init(0), cl::Hidden, cl::desc("Maximum SLP vectorization factor (0=unlimited)"))
static cl::opt< int >	ScheduleRegionSizeBudget ("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block"))
	Limits the size of scheduling regions in a block.
static cl::opt< int >	MinVectorRegSizeOption ("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static cl::opt< unsigned >	RecursionMaxDepth ("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree"))
static cl::opt< unsigned >	MinTreeSize ("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable"))
static cl::opt< unsigned >	PHINodeVectorizationBudget ("slp-phi-vectorization-budget", cl::init(1024), cl::Hidden, cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs"))
static cl::opt< int >	LookAheadMaxDepth ("slp-max-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for operand reordering scores"))
static cl::opt< int >	RootLookAheadMaxDepth ("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for searching best rooting option"))
static cl::opt< unsigned >	MinProfitableStridedLoads ("slp-min-strided-loads", cl::init(2), cl::Hidden, cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value"))
static cl::opt< unsigned >	MinProfitableStridedStores ("slp-min-strided-stores", cl::init(2), cl::Hidden, cl::desc("The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value"))
static cl::opt< unsigned >	MaxProfitableStride ("slp-max-stride", cl::init(8), cl::Hidden, cl::desc("The maximum stride, considered to be profitable."))
static cl::opt< bool >	EnableStridedStores ("slp-enable-strided-stores", cl::init(false), cl::Hidden, cl::desc("Enable SLP trees to be built from strided " "store chains."))
static cl::opt< bool >	EnableMaskedStores ("slp-enable-masked-stores", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores."))
static cl::opt< bool >	DisableTreeReorder ("slp-disable-tree-reorder", cl::init(false), cl::Hidden, cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only."))
static cl::opt< bool >	ForceStridedLoads ("slp-force-strided-loads", cl::init(false), cl::Hidden, cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only."))
static cl::opt< bool >	ViewSLPTree ("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz"))
static cl::opt< bool >	VectorizeNonPowerOf2 ("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, cl::desc("Try to vectorize with non-power-of-2 number of elements."))
static cl::opt< bool >	ForcePostProcessStoresOperands ("slp-postprocess-stores-operands", cl::init(false), cl::Hidden, cl::desc("Force vectorization of non-vectorizable stores operands."))
static cl::opt< bool >	NonVectReductions ("slp-non-vectorizables-as-reductions", cl::init(false), cl::Hidden, cl::desc("Use non-vectorizable instructions as potential reduction roots."))
static cl::opt< bool >	VectorizePoorThroughput ("slp-vectorize-poor-throughput", cl::init(true), cl::Hidden, cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds."))
static cl::opt< bool >	VectorizeCopyableElements ("slp-copyable-elements", cl::init(true), cl::Hidden, cl::desc("Try to replace values with the idempotent instructions for " "better vectorization."))
	Enables vectorization of copyable elements.
static cl::opt< unsigned >	LoopAwareTripCount ("slp-cost-loop-trip-count", cl::init(2), cl::Hidden, cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)"))
static cl::opt< bool >	PerLaneGatherScale ("slp-per-lane-gather-scale", cl::init(true), cl::Hidden, cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences."))
	Refine the loop-aware cost scaling of gather/buildvector tree entries by using the per-lane execution scale of the operand that feeds each lane, instead of a single whole-entry scale.
static const unsigned	AliasedCheckLimit = 10
static constexpr int	UsesLimit = 64
static const unsigned	MaxMemDepDistance = 160
static const int	MinScheduleRegionSize = 16
	If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.
static const unsigned	MaxPHINumOperands = 128
	Maximum allowed number of operands in the PHI nodes.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "SLP"

Definition at line 114 of file SLPVectorizer.cpp.

SV_NAME

#define SV_NAME   "slp-vectorizer"

Definition at line 113 of file SLPVectorizer.cpp.

Function Documentation

addMask()

void addMask ( SmallVectorImpl< int > & Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs = false )

static

Shuffles Mask in accordance with the given SubMask.

Parameters

ExtendingManyInputs

Supports reshuffling of the mask with not only one but two input vectors.

Definition at line 1777 of file SLPVectorizer.cpp.

References assert(), llvm::ArrayRef< T >::begin(), E(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

allStructUsersAreExtractValueInsts()

bool allStructUsersAreExtractValueInsts ( ArrayRef< Value * > VL )

static

Definition at line 10897 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::isa(), and llvm::IsaPred.

areAllOperandsNonInsts()

bool areAllOperandsNonInsts ( Value * V )

static

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all operands are either not instructions or phi nodes or instructions from different blocks.

Definition at line 1888 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and llvm::mayHaveNonDefUseDependency().

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

areCompatibleCmpOps()

bool areCompatibleCmpOps ( Value * BaseOp0, Value * BaseOp1, Value * Op0, Value * Op1, const TargetLibraryInfo & TLI )

static

Checks if the provided operands of 2 cmp instructions are compatible, i.e.

compatible instructions or constants, or just some other regular values.

Definition at line 1493 of file SLPVectorizer.cpp.

References getSameOpcode(), llvm::isa(), and llvm::slpvectorizer::isConstant().

Referenced by isCmpSameOrSwapped().

arePointersCompatible()

bool arePointersCompatible ( Value * Ptr1, Value * Ptr2, const TargetLibraryInfo & TLI, bool CompareOpcodes = true )

static

Definition at line 6870 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), getSameOpcode(), llvm::getUnderlyingObject(), llvm::slpvectorizer::isConstant(), and RecursionMaxDepth.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

areTwoInsertFromSameBuildVector()

bool areTwoInsertFromSameBuildVector ( InsertElementInst * VU, InsertElementInst * V, function_ref< Value *(InsertElementInst *)> GetBaseOperand )

static

Check if two insertelement instructions are from the same buildvector.

Definition at line 8219 of file SLPVectorizer.cpp.

References llvm::dyn_cast_or_null(), llvm::VectorType::getElementCount(), getElementIndex(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::InsertElementInst::getType(), llvm::Value::hasOneUse(), llvm::SmallBitVector::set(), and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost(), and slpvectorizer::BoUpSLP::vectorizeTree().

buildCompressMask()

bool buildCompressMask ( ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, Type * ScalarTy, const DataLayout & DL, ScalarEvolution & SE, SmallVectorImpl< int > & CompressMask )

static

Builds compress-like mask for shuffles for the given PointerOps, ordered with Order.

Returns

true if the mask represents strided access, false - otherwise.

Definition at line 7167 of file SLPVectorizer.cpp.

References llvm::SmallVectorImpl< T >::assign(), DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), llvm::getPointersDiff(), I, llvm::PoisonMaskElem, llvm::seq(), and llvm::ArrayRef< T >::size().

Referenced by isMaskedLoadCompress().

buildIntrinsicArgTypes()

SmallVector< Type * > buildIntrinsicArgTypes ( const CallInst * CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW, const TargetTransformInfo * TTI )

static

Builds the arguments types vector for the given call instruction with the given ID for the specified vector factor.

Definition at line 10613 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), llvm::enumerate(), llvm::IntegerType::get(), llvm::Value::getContext(), getWidenedType(), llvm::isVectorIntrinsicWithScalarOpAtArg(), llvm::Intrinsic::not_intrinsic, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by getVectorOpCost().

buildUseMask()

SmallBitVector buildUseMask ( int VF, ArrayRef< int > Mask, UseMask MaskArg )

static

Prepares a use bitset for the given mask either for the first argument or for the second.

Definition at line 666 of file SLPVectorizer.cpp.

References llvm::enumerate(), and llvm::PoisonMaskElem.

Referenced by performExtractsShuffleAction().

calculateRtStride()

const SCEV * calculateRtStride ( ArrayRef< Value * > PointerOps, Type * ElemTy, const DataLayout & DL, ScalarEvolution & SE, SmallVectorImpl< unsigned > & SortedIndices )

static

Checks if the provided list of pointers Pointers represents the strided pointers for type ElemTy.

If they are not, nullptr is returned. Otherwise, SCEV* of the stride value is returned. If PointerOps can be rearanged into the following sequence:

%x + c_0 * stride,
%x + c_1 * stride,
%x + c_2 * stride
...

where each c_i is constant. The SCEV of the stride will be returned.

Definition at line 6917 of file SLPVectorizer.cpp.

References llvm::SmallVectorImpl< T >::clear(), DL, llvm::dyn_cast(), llvm::enumerate(), llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getMinusSCEV(), llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getType(), llvm::ScalarEvolution::getUDivExactExpr(), llvm::isa(), llvm::SCEV::isNonConstantNegative(), llvm::SCEV::isZero(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), Size, llvm::ArrayRef< T >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate().

calculateShufflevectorMask()

SmallVector< int > calculateShufflevectorMask ( ArrayRef< Value * > VL )

static

Returns

a shufflevector mask which is used to vectorize shufflevectors e.g., %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> the result is <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>

Definition at line 494 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), llvm::ArrayRef< T >::front(), getShufflevectorNumGroups(), and llvm::PoisonMaskElem.

canConvertToFMA()

InstructionCost canConvertToFMA ( ArrayRef< Value * > VL, const InstructionsState & S, DominatorTree & DT, const DataLayout & DL, TargetTransformInfo & TTI, const TargetLibraryInfo & TLI )

static

Check if we can convert fadd/fsub sequence to FMAD.

Returns

Cost of the FMAD, if conversion is possible, invalid cost otherwise.

Definition at line 14713 of file SLPVectorizer.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, llvm::all_of(), llvm::FastMathFlags::allowContract(), Analysis, assert(), CostKind, DL, llvm::dyn_cast(), llvm::ArrayRef< T >::front(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::InstructionCost::getInvalid(), getSameOpcode(), I, llvm::FastMathFlags::set(), llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::zip().

checkEVsForVecCalls()

bool checkEVsForVecCalls ( ArrayRef< Value * > VL, const InstructionsState & S, const TargetLibraryInfo & TLI, SmallVectorImpl< unsigned > & Indices, SmallVectorImpl< Value * > & Calls )

static

Detects an extractvalue bundle that can be widened by vectorizing the underlying struct-returning calls.

VL is a bundle whose state S is Instruction::ExtractValue. The bundle is acceptable for widening into one struct-of-vectors call only when:

• every element of VL is an ExtractValueInst,
• every ExtractValueInst extracts the same struct field (its getIndices() matches the main op's indices),
• the aggregate operands form a uniform set of CallInsts (per getSameOpcode) that is not an alt-shuffle and whose return type is a literal struct, and
• every user of every such call is itself an ExtractValueInst, so the external-use extraction code can rebuild scalars via extractvalue + extractelement without needing an insertvalue chain.

On success returns true and fills Indices with the common field index path and Calls with the per-lane aggregate calls (in VL order), for the caller to feed as the operand of the new tree entry. Otherwise returns false and leaves the output parameters untouched.

Definition at line 10853 of file SLPVectorizer.cpp.

References Aggregates, llvm::all_of(), llvm::slpvectorizer::allSameBlock(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::cast(), llvm::ArrayRef< T >::end(), getSameOpcode(), llvm::isa(), llvm::IsaPred, IV, and llvm::SmallVectorImpl< T >::swap().

clusterSortPtrAccesses()

bool clusterSortPtrAccesses ( ArrayRef< Value * > VL, ArrayRef< BasicBlock * > BBs, Type * ElemTy, const DataLayout & DL, ScalarEvolution & SE, SmallVectorImpl< unsigned > & SortedIndices )

static

Definition at line 8089 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::sampleprof::Base, llvm::SmallVectorImpl< T >::clear(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::Depth, DL, llvm::ArrayRef< T >::drop_front(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::enumerate(), llvm::ArrayRef< T >::front(), llvm::getUnderlyingObject(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::SmallPtrSetImpl< PtrType >::insert(), P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RecursionMaxDepth, llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::stable_sort(), T, and llvm::V1.

Referenced by slpvectorizer::BoUpSLP::findPartiallyOrderedLoads().

combineOrders()

void combineOrders ( MutableArrayRef< unsigned > Order, ArrayRef< unsigned > SecondaryOrder )

static

Definition at line 8674 of file SLPVectorizer.cpp.

References assert(), llvm::ArrayRef< T >::empty(), llvm::seq(), llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

compareCmp()

template<bool IsCompatibility>

bool compareCmp ( Value * V, Value * V2, TargetLibraryInfo & TLI, const DominatorTree & DT )

static

Compare two cmp instructions.

If IsCompatibility is true, function returns true if 2 cmps have same/swapped predicates and mos compatible corresponding operands. If IsCompatibility is false, function implements strict weak ordering relation between two cmp instructions, returning true if the first instruction is "less" than the second, i.e. its predicate is less than the predicate of the second or the operands IDs are less than the operands IDs of the second cmp instruction.

Definition at line 32087 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), E(), llvm::DomTreeNodeBase< NodeT >::getDFSNumIn(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), llvm::User::getOperand(), getSameOpcode(), llvm::Type::getScalarSizeInBits(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::Type::getTypeID(), I, and isValidElementType().

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts().

computeCommonAlignment()

template<typename T>

Align computeCommonAlignment ( ArrayRef< Value * > VL )

static

Calculates minimal alignment as a common alignment.

Definition at line 6889 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::ArrayRef< T >::consume_front(), and getAlign().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

createExtractVector()

Value * createExtractVector ( IRBuilderBase & Builder, Value * Vec, unsigned SubVecVF, unsigned Index )

static

Generates subvector extract using Generator or using default shuffle.

Definition at line 7157 of file SLPVectorizer.cpp.

References llvm::PoisonMaskElem.

Referenced by slpvectorizer::BoUpSLP::vectorizeTree().

createInsertVector()

Value * createInsertVector ( IRBuilderBase & Builder, Value * Vec, Value * V, unsigned Index, function_ref< Value *(Value *, Value *, ArrayRef< int >)> Generator = {} )

static

Creates subvector insert.

Generates shuffle using Generator or using default shuffle.

Definition at line 7127 of file SLPVectorizer.cpp.

DEBUG_COUNTER()

DEBUG_COUNTER	(	VectorizedGraphs	,
		"slp-vectorized"	,
		"Controls which SLP graphs should be vectorized."	)

doesInTreeUserNeedToExtract()

bool doesInTreeUserNeedToExtract ( Value * Scalar, Instruction * UserInst, TargetLibraryInfo * TLI, const TargetTransformInfo * TTI )

static

Returns

True if in-tree use also needs extract. This refers to possible scalar operand in vectorized instruction.

Definition at line 1726 of file SLPVectorizer.cpp.

References llvm::any_of(), llvm::CallBase::args(), llvm::cast(), llvm::enumerate(), llvm::Instruction::getOpcode(), llvm::LoadInst::getPointerOperand(), and llvm::getVectorIntrinsicIDForCall().

Referenced by slpvectorizer::BoUpSLP::buildExternalUses().

doesNotNeedToBeScheduled()

bool doesNotNeedToBeScheduled ( Value * V )

static

Checks if the specified value does not require scheduling.

It does not require scheduling if all operands and all users do not need to be scheduled in the current basic block.

Definition at line 1922 of file SLPVectorizer.cpp.

References areAllOperandsNonInsts(), and isUsedOutsideBlock().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

doesNotNeedToSchedule()

bool doesNotNeedToSchedule ( ArrayRef< Value * > VL )

static

Checks if the specified array of instructions does not require scheduling.

It is so if all either instructions have operands that do not require scheduling or their users do not require scheduling since they are phis or in other basic blocks.

Definition at line 1930 of file SLPVectorizer.cpp.

References llvm::all_of(), areAllOperandsNonInsts(), llvm::ArrayRef< T >::empty(), and isUsedOutsideBlock().

Referenced by tryToFindDuplicates().

dumpOrder()

LLVM_DUMP_METHOD void dumpOrder ( const BoUpSLP::OrdersType & Order )

static

Definition at line 9845 of file SLPVectorizer.cpp.

References llvm::dbgs(), and LLVM_DUMP_METHOD.

findBuildAggregate()

bool findBuildAggregate ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, const BoUpSLP & R )

static

Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.

Also recognize homogeneous aggregates like {<2 x float>, <2 x float>}, {{float, float}, {float, float}}, [2 x {float, float}] and so on. See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples.

Assume LastInsertInst is of InsertElementInst or InsertValueInst type.

Returns

true if it matches.

Definition at line 31519 of file SLPVectorizer.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::erase(), findBuildAggregateRec(), getAggregateSize(), llvm::isa(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

findBuildAggregateRec()

void findBuildAggregateRec ( Instruction * LastInsertInst, TargetTransformInfo * TTI, SmallVectorImpl< Value * > & BuildVectorOpds, SmallVectorImpl< Value * > & InsertElts, unsigned OperandOffset, const BoUpSLP & R )

static

Definition at line 31480 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), findBuildAggregateRec(), getElementIndex(), llvm::User::getOperand(), llvm::Value::hasOneUse(), and llvm::isa().

Referenced by findBuildAggregate(), and findBuildAggregateRec().

findInnermostNonInvariantLoop()

const Loop * findInnermostNonInvariantLoop ( const Loop * L, ArrayRef< Value * > VL )

static

Find the innermost loop starting from L, for which at least a single value in VL is not invariant.

Definition at line 10802 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), and llvm::isa().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

findInstructionWithOpcode()

Instruction * findInstructionWithOpcode ( ArrayRef< Value * > VL, unsigned Opcode )

static

Find an instruction with a specific opcode in VL.

Parameters

VL	Array of values to search through. Must contain only Instructions and PoisonValues.
Opcode	The instruction opcode to search for

Returns

• The first instruction found with matching opcode
• nullptr if no matching instruction is found

Definition at line 1478 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), and llvm::isa().

Referenced by getSameOpcode().

fixupOrderingIndices()

void fixupOrderingIndices ( MutableArrayRef< unsigned > Order )

static

Order may have elements assigned special value (size) which is out of bounds.

Such indices only appear on places which correspond to undef values (see canReuseExtract for details) and used in order to avoid undef values have effect on operands ordering. The first loop below simply finds all unused indices and then the next loop nest assigns these indices for undef values positions. As an example below Order has two undef positions and they have assigned values 3 and 7 respectively: before: 6 9 5 4 9 2 1 0 after: 6 3 5 4 7 2 1 0

Definition at line 1812 of file SLPVectorizer.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::SmallBitVector::find_first(), llvm::SmallBitVector::find_next(), I, llvm::SmallBitVector::none(), llvm::SmallBitVector::reset(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

forEachOperandChainCandidate()

template<typename Func>

void forEachOperandChainCandidate ( Instruction * I, Func F, bool ForReduction )

static

Visits the value operands of I that are candidates for operand-chain vectorization.

Definition at line 32280 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::enumerate(), F, I, llvm_unreachable, and NonVectReductions.

Referenced by llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

gatherPossiblyVectorizableLoads()

void gatherPossiblyVectorizableLoads ( const BoUpSLP & R, ArrayRef< Value * > VL, const DataLayout & DL, ScalarEvolution & SE, const TargetTransformInfo & TTI, SmallVectorImpl< SmallVector< std::pair< LoadInst *, int64_t > > > & GatheredLoads, bool AddNew = true )

static

Tries to find subvector of loads and builds new vector of only loads if can be profitable.

Definition at line 9904 of file SLPVectorizer.cpp.

References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::bit_ceil(), llvm::SetVector< T, Vector, Set, N >::contains(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::SmallSet< T, N, C >::contains(), llvm::Data, DL, llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::ArrayRef< T >::empty(), llvm::SetVector< T, Vector, Set, N >::empty(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::find_if(), llvm::ArrayRef< T >::front(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::LoadInst::getPointerOperand(), llvm::getPointersDiff(), llvm::Value::getType(), llvm::getUnderlyingObject(), getValueType(), llvm::has_single_bit(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SmallSet< T, N, C >::insert(), llvm::SetVector< T, Vector, Set, N >::insert_range(), isValidElementType(), llvm::Offset, P, RecursionMaxDepth, llvm::seq(), llvm::ArrayRef< T >::size(), and llvm::zip().

generateKeySubkey()

std::pair< size_t, size_t > generateKeySubkey ( Value * V, const TargetLibraryInfo * TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate )

static

Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.

The keys/subkeys can be used for better sorting of the values themselves (keys) and in values subgroups (subkeys).

Definition at line 10522 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), Call, llvm::cast(), llvm::dyn_cast(), generateKeySubkey(), llvm::CmpInst::getInversePredicate(), llvm::VFDatabase::getMappings(), llvm::CmpInst::getSwappedPredicate(), llvm::getVectorIntrinsicIDForCall(), llvm::hash_combine(), llvm::hash_value(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), llvm::Instruction::isIntDivRem(), llvm::isTriviallyVectorizable(), isUndefVector(), isValidForAlternation(), and llvm::slpvectorizer::isVectorLikeInstWithConstOps().

Referenced by generateKeySubkey().

getAggregateSize()

std::optional< unsigned > getAggregateSize ( Instruction * InsertInst )

static

Definition at line 31452 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::Type::isSingleValueType(), and IV.

Referenced by findBuildAggregate().

getAltInstrMask()

SmallBitVector getAltInstrMask ( ArrayRef< Value * > VL, Type * ScalarTy, unsigned Opcode0, unsigned Opcode1 )

static

Returns

a bitset for selecting opcodes. false for Opcode0 and true for Opcode1.

Definition at line 1838 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::slpvectorizer::getNumElements(), llvm::isa(), llvm::seq(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

getDebugLocFromPHI()

DebugLoc getDebugLocFromPHI ( PHINode & PN )

static

Definition at line 23257 of file SLPVectorizer.cpp.

References DL, llvm::Instruction::getDebugLoc(), and llvm::DebugLoc::getUnknown().

getElementIndex()

std::optional< unsigned > getElementIndex ( const Value * Inst, unsigned Offset = 0 )

static

Returns

inserting or extracting index of InsertElement, ExtractElement or InsertValue instruction, using Offset as base offset for index.

std::nullopt if the index is not an immediate.

Definition at line 618 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::slpvectorizer::getInsertExtractIndex(), I, IV, and llvm::Offset.

Referenced by areTwoInsertFromSameBuildVector(), and slpvectorizer::BoUpSLP::getReorderingData().

getExtractWithExtendCost()

InstructionCost getExtractWithExtendCost ( const TargetTransformInfo & TTI, unsigned Opcode, Type * Dst, VectorType * VecTy, unsigned Index, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput )

static

This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst is a FixedVectorType, a vector will be extracted instead of a scalar.

Definition at line 7109 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), CostKind, llvm::slpvectorizer::getNumElements(), getShuffleCost(), getWidenedType(), llvm::isVectorizedTy(), llvm::TargetTransformInfo::None, llvm::TargetTransformInfo::SK_ExtractSubvector, SLPReVec, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

getFloorFullVectorNumberOfElements()

unsigned getFloorFullVectorNumberOfElements ( const TargetTransformInfo & TTI, Type * Ty, unsigned Sz )

static

Returns the number of elements of the given type Ty, not greater than Sz, which forms type, which splits by TTI into whole vector types during legalization.

Definition at line 389 of file SLPVectorizer.cpp.

References llvm::bit_ceil(), llvm::bit_floor(), llvm::divideCeil(), getWidenedType(), llvm::isa(), and isValidElementType().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

getFullVectorNumberOfElements()

unsigned getFullVectorNumberOfElements ( const TargetTransformInfo & TTI, Type * Ty, unsigned Sz )

static

Returns the number of elements of the given type Ty, not less than Sz, which forms type, which splits by TTI into whole vector types during legalization.

Definition at line 374 of file SLPVectorizer.cpp.

References llvm::bit_ceil(), llvm::divideCeil(), getWidenedType(), llvm::isa(), and isValidElementType().

Referenced by getNumberOfParts(), and isMaskedLoadCompress().

getGEPCosts()

std::pair< InstructionCost, InstructionCost > getGEPCosts ( const TargetTransformInfo & TTI, ArrayRef< Value * > Ptrs, Value * BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind, Type * ScalarTy, VectorType * VecTy )

static

Calculate the scalar and the vector costs from vectorizing set of GEPs.

Definition at line 14468 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::Value::hasOneUse(), llvm::IsaPred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::TargetTransformInfo::TCC_Free.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

getLocation()

MemoryLocation getLocation ( Instruction * I )

static

Returns

the AA location that is being access by the instruction.

Definition at line 1755 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::MemoryLocation::get(), and I.

getLoopTripCount()

unsigned getLoopTripCount ( const Loop * L, ScalarEvolution & SE )

static

Get the assumed loop trip count for the loop L.

Definition at line 16703 of file SLPVectorizer.cpp.

References llvm::getLoopEstimatedTripCount(), llvm::ScalarEvolution::getSmallConstantTripCount(), and LoopAwareTripCount.

getMainAltOpsNoStateVL()

std::pair< Instruction *, Instruction * > getMainAltOpsNoStateVL ( ArrayRef< Value * > VL )

static

Returns main/alternate instructions for the given VL.

Unlike getSameOpcode supports non-compatible instructions for better SplitVectorize node support.

Returns

first main/alt instructions, if only poisons and instruction with only 2 opcodes exists. Returns pair of nullptr otherwise.

Definition at line 11473 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), I, and llvm::isa().

getNonPhiOperand()

Instruction * getNonPhiOperand ( Instruction * I, PHINode * Phi )

static

Returns the first operand of I that does not match Phi.

If operand is not an instruction it returns nullptr.

Definition at line 31642 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), I, and matchRdxBop().

getNumberOfParts()

unsigned getNumberOfParts ( const TargetTransformInfo & TTI, Type * VecTy, Type * ScalarTy, const unsigned Limit = std::numeric_limits<unsigned>::max() )

static

Returns number of parts, the type VecTy will be split at the codegen phase.

If the type is going to be scalarized or does not uses whole registers, returns 1.

Definition at line 1957 of file SLPVectorizer.cpp.

References llvm::divideCeil(), getFullVectorNumberOfElements(), llvm::slpvectorizer::getNumElements(), hasFullVectorsOrPowerOf2(), llvm::isa(), and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::computeMinimumValueSizes(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::optimizeGatherSequence(), slpvectorizer::BoUpSLP::processBuildVector(), and tryToFindDuplicates().

getNumberOfPotentiallyCommutativeOps()

unsigned getNumberOfPotentiallyCommutativeOps ( Instruction * I )

static

Returns

number of operands of I, considering commutativity. Returns 2 for commutative intrinsics.

Parameters

I	The instruction to check for commutativity

Definition at line 605 of file SLPVectorizer.cpp.

References I, llvm::isa(), and isCommutative().

getPoorThroughputTypeKey()

unsigned getPoorThroughputTypeKey ( Type * Ty )

static

Packs a type's kind and scalar width into one key, so an opcode/intrinsic id reused across scalar widths (e.g.

fdiv float vs. fdiv double) is cached per width instead of sharing one verdict.

Definition at line 10704 of file SLPVectorizer.cpp.

Referenced by isPoorThroughputOp().

getRdxKind()

RecurKind getRdxKind ( Value * V )

static

Gets recurrence kind from the specified value.

Definition at line 31449 of file SLPVectorizer.cpp.

getReductionInstr()

Instruction * getReductionInstr ( const DominatorTree * DT, PHINode * P, BasicBlock * ParentBB, LoopInfo * LI )

static

Try and get a reduction instruction from a phi node.

Given a phi node P in a block ParentBB, consider possible reductions if they come from either ParentBB or a containing loop latch.

Returns

A candidate reduction value if possible, or

nullptr 

if not possible.

Definition at line 31554 of file SLPVectorizer.cpp.

References llvm::cast(), llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::isa(), and P.

getSameOpcode()

InstructionsState getSameOpcode ( ArrayRef< Value * > VL, const TargetLibraryInfo & TLI )

static

Returns

analysis of the Instructions in VL described in InstructionsState, the Opcode that we suppose the whole list could be vectorized even if its structure is diverse.

Definition at line 1529 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::slpvectorizer::allSameOpcode(), assert(), llvm::CmpInst::BAD_ICMP_PREDICATE, Call, llvm::cast(), llvm::SetVector< T, Vector, Set, N >::contains(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::find_if(), findInstructionWithOpcode(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::CallBase::getBundleOperandsStartIndex(), llvm::CallBase::getCalledFunction(), llvm::VFDatabase::getMappings(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::getVectorIntrinsicIDForCall(), llvm::CallBase::hasOperandBundles(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::isa(), llvm::IsaPred, isCmpSameOrSwapped(), llvm::isTriviallyVectorizable(), isValidForAlternation(), llvm::slpvectorizer::isVectorLikeInstWithConstOps(), llvm::iterator_range(), llvm::User::op_begin(), llvm::ArrayRef< T >::size(), llvm::SetVector< T, Vector, Set, N >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by areCompatibleCmpOps(), arePointersCompatible(), canConvertToFMA(), checkEVsForVecCalls(), compareCmp(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::VLOperands::reorder().

getScalarizationOverhead()

InstructionCost getScalarizationOverhead ( const TargetTransformInfo & TTI, Type * ScalarTy, VectorType * Ty, const APInt & DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc = true, ArrayRef< Value * > VL = {}, TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None )

static

This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorType, a vector will be inserted or extracted instead of a scalar.

Definition at line 7055 of file SLPVectorizer.cpp.

References CostKind.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), llvm::LoopVectorizationCostModel::getDivRemSpeculationCost(), llvm::LoopVectorizationCostModel::getVectorCallCost(), isMaskedLoadCompress(), and tryToFindDuplicates().

getShuffleCost()

InstructionCost getShuffleCost ( const TargetTransformInfo & TTI, TTI::ShuffleKind Kind, VectorType * Tp, ArrayRef< int > Mask = {}, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, int Index = 0, VectorType * SubTp = nullptr, ArrayRef< const Value * > Args = {} )

static

Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.

Adds SLP-specifc cost estimation for insert subvector pattern.

Definition at line 7027 of file SLPVectorizer.cpp.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canVectorizeLoads(), getExtractWithExtendCost(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getTreeCost(), getVectorInstrCost(), isMaskedLoadCompress(), and tryToFindDuplicates().

getShufflevectorNumGroups()

unsigned getShufflevectorNumGroups ( ArrayRef< Value * > VL )

static

Returns

the number of groups of shufflevector A group has the following features

1. All of value in a group are shufflevector.
2. The mask of all shufflevector is isExtractSubvectorMask.
3. The mask of all shufflevector uses all of the elements of the source. e.g., it is 1 group (%0) %1 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %2 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> it is 2 groups (%3 and %4) %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> it is 0 group %12 = shufflevector <8 x i16> %10, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %13 = shufflevector <8 x i16> %11, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>

Definition at line 442 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), llvm::all_of(), assert(), llvm::cast(), E(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), I, llvm::IsaPred, llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().

Referenced by calculateShufflevectorMask().

getValueType()

Type * getValueType ( Value * V, bool LookThroughCmp = false )

static

Returns the "element type" of the given value/instruction V.

For stores, returns the stored value type; for insertelement (when ReVec is off), the inserted operand type. For compares, the default is to return the result type (i1); when LookThroughCmp is true, returns the type of the compared operands instead, which is needed for vector width calculations (the width is determined by the operand type, not the i1 result).

Definition at line 333 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), IV, and SLPReVec.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), CollectOpsToWiden(), combineArithReduction(), combineBitcastvxi1(), combineMulToPMULDQ(), combineSetCC(), combineToHorizontalAddSub(), CompactSwizzlableVector(), detectPMADDUBSW(), FoldBUILD_VECTOR(), foldCONCAT_VECTORS(), gatherPossiblyVectorizableLoads(), llvm::TargetLoweringBase::getAsmOperandValueType(), llvm::Function::getFunctionType(), slpvectorizer::BoUpSLP::getInsertBuildVectorSrcTy(), llvm::RISCVTTIImpl::getIntImmCostInst(), llvm::BasicTTIImplBase< BasicTTIImpl >::getIntrinsicInstrCost(), llvm::TargetLoweringBase::getMemValueType(), slpvectorizer::BoUpSLP::getReorderingData(), llvm::TargetLoweringBase::getSimpleValueType(), getTestBitOperand(), slpvectorizer::BoUpSLP::getTreeCost(), llvm::TargetLoweringBase::isExtLoad(), LLVMGlobalGetValueType(), matchPMADDWD(), matchPMADDWD_2(), mayFoldIntoVector(), narrowVectorSelect(), performCONCAT_VECTORSCombine(), slpvectorizer::BoUpSLP::reorderBottomToTop(), slpvectorizer::BoUpSLP::reorderTopToBottom(), llvm::SelectionDAGISel::SelectInlineAsmMemoryOperands(), tryToFindDuplicates(), tryToVectorizeSequence(), and llvm::SLPVectorizerPass::vectorizeCmpInsts().

getVectorCallCosts()

std::pair< InstructionCost, InstructionCost > getVectorCallCosts ( CallInst * CI, Type * VecTy, const TargetTransformInfo * TTI, const TargetLibraryInfo * TLI, ArrayRef< Type * > ArgTys )

static

Calculates the costs of vectorized intrinsic (if possible) and vectorized function (if possible) calls.

Returns invalid cost for the corresponding calls, if they cannot be vectorized/will be scalarized.

Definition at line 10638 of file SLPVectorizer.cpp.

References llvm::CallBase::args(), llvm::dyn_cast(), llvm::VFShape::get(), llvm::ElementCount::getFixed(), llvm::CallBase::getFunctionType(), llvm::InstructionCost::getInvalid(), llvm::slpvectorizer::getNumElements(), llvm::getVectorIntrinsicIDForCall(), llvm::VFDatabase::getVectorizedFunction(), IntrinsicCost, llvm::CallBase::isNoBuiltin(), and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by getVectorOpCost().

getVectorInstrContextHint()

TTI::VectorInstrContext getVectorInstrContextHint ( ArrayRef< Value * > VL, const APInt & DemandedElts )

static

Calculates a VectorInstrContext from the values in VL at indices in DemandedElts.

Definition at line 16997 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), llvm::TargetTransformInfo::getVectorInstrContextHint(), I, llvm::isa(), llvm::TargetTransformInfo::None, llvm::seq(), and llvm::ArrayRef< T >::size().

getVectorInstrCost()

InstructionCost getVectorInstrCost ( const TargetTransformInfo & TTI, Type * ScalarTy, unsigned Opcode, Type * Val, TTI::TargetCostKind CostKind, unsigned Index, Value * Scalar, ArrayRef< std::tuple< Value *, User *, int > > ScalarUserAndIdx )

static

This is similar to TargetTransformInfo::getVectorInstrCost, but if ScalarTy is a FixedVectorType, a vector will be extracted instead of a scalar.

Definition at line 7090 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), CostKind, llvm::dyn_cast(), getShuffleCost(), llvm::isa(), llvm::TargetTransformInfo::SK_ExtractSubvector, and SLPReVec.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

getVectorOpCost()

InstructionCost getVectorOpCost ( Instruction * I, unsigned VF, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI )

static

Returns

the reciprocal-throughput cost of I widened to VF lanes (an arithmetic op or a vectorizable call).

Definition at line 10684 of file SLPVectorizer.cpp.

References assert(), buildIntrinsicArgTypes(), CostKind, llvm::dyn_cast(), getVectorCallCosts(), llvm::getVectorIntrinsicIDForCall(), getWidenedType(), I, llvm::isa(), and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by isPoorThroughputOp().

getWidenedType()

Type * getWidenedType ( Type * ScalarTy, unsigned VF )

static

Returns

the vector type of ScalarTy based on vectorization factor.

Definition at line 348 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), llvm::dyn_cast(), llvm::FixedVectorType::get(), llvm::StructType::get(), llvm::ElementCount::getFixed(), llvm::slpvectorizer::getNumElements(), llvm::isUnpackedStructLiteral(), llvm::VectorType::isValidElementType(), llvm::isVectorizedTy(), llvm::map_to_vector(), llvm::toScalarizedTy(), and llvm::toVectorizedTy().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::analyzeConstantStrideCandidate(), slpvectorizer::BoUpSLP::analyzeRtStrideCandidate(), buildIntrinsicArgTypes(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canMapToVector(), slpvectorizer::BoUpSLP::canVectorizeLoads(), slpvectorizer::BoUpSLP::computeMinimumValueSizes(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), getExtractWithExtendCost(), getFloorFullVectorNumberOfElements(), getFullVectorNumberOfElements(), slpvectorizer::BoUpSLP::getInsertBuildVectorSrcTy(), slpvectorizer::BoUpSLP::getReductionType(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getSpillCost(), slpvectorizer::BoUpSLP::getTreeCost(), getVectorOpCost(), hasFullVectorsOrPowerOf2(), isMaskedLoadCompress(), isMaskedStoreCompress(), slpvectorizer::BoUpSLP::isStridedLoad(), slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::needToDelay(), slpvectorizer::BoUpSLP::optimizeGatherSequence(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderTopToBottom(), tryToFindDuplicates(), and slpvectorizer::BoUpSLP::vectorizeTree().

hasFullVectorsOrPowerOf2()

bool hasFullVectorsOrPowerOf2 ( const TargetTransformInfo & TTI, Type * Ty, unsigned Sz )

static

Returns true if widened type of Ty elements with size Sz represents full vector type, i.e.

adding extra element results in extra parts upon type legalization.

Definition at line 1938 of file SLPVectorizer.cpp.

References getWidenedType(), llvm::has_single_bit(), llvm::isa(), and isValidElementType().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and getNumberOfParts().

inversePermutation()

void inversePermutation ( ArrayRef< unsigned > Indices, SmallVectorImpl< int > & Mask )

static

Definition at line 1863 of file SLPVectorizer.cpp.

References E(), I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::addOrdered(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getReorderingData(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

isAllowedNonPowerOf2VF()

bool isAllowedNonPowerOf2VF ( unsigned NumElts )

static

True when slp-vectorize-non-power-of-2 is enabled and NumElts is a supported non-power-of-2 width: NumElts + 1 must be a power of two (e.g.

3 or 7 lanes, i.e. almost a full power-of-2 register).

Definition at line 265 of file SLPVectorizer.cpp.

References llvm::has_single_bit(), and VectorizeNonPowerOf2.

isAlternateInstruction()

bool isAlternateInstruction ( Instruction * I, Instruction * MainOp, Instruction * AltOp, const TargetLibraryInfo & TLI )

static

Checks if the specified instruction I is an alternate operation for the given MainOp and AltOp instructions.

Definition at line 14044 of file SLPVectorizer.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), llvm::CmpInst::getSwappedPredicate(), I, isCmpSameOrSwapped(), and P.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and slpvectorizer::BoUpSLP::getReorderingData().

isCmpSameOrSwapped()

bool isCmpSameOrSwapped ( const CmpInst * BaseCI, const CmpInst * CI, const TargetLibraryInfo & TLI )

static

Returns

true if a compare instruction CI has similar "look" and same predicate as BaseCI, "as is" or with its operands and predicate swapped, false otherwise.

Definition at line 1507 of file SLPVectorizer.cpp.

References areCompatibleCmpOps(), assert(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::CmpInst::getSwappedPredicate(), and llvm::Value::getType().

Referenced by getSameOpcode(), and isAlternateInstruction().

isCommutableOperand()

bool isCommutableOperand ( const Instruction * I, Value * ValWithUses, unsigned Op, bool IsCopyable = false )

static

Checks if the operand is commutative.

In commutative operations, not all operands might commutable, e.g. for fmuladd only 2 first operands are commutable.

Definition at line 574 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), I, llvm::isa(), and isCommutative().

isCommutative() [1/2]

bool isCommutative ( const Instruction * I )

static

This is a helper function to check whether I is commutative.

This is a convenience wrapper that calls the two-parameter version of isCommutative with the same instruction for both parameters. This is the common case where the instruction being checked for commutativity is the same as the instruction whose uses are analyzed for special patterns (see the two-parameter version above for details).

Parameters

I	The instruction to check for commutativity

Returns

true if the instruction is commutative, false otherwise

Definition at line 600 of file SLPVectorizer.cpp.

References I, and isCommutative().

isCommutative() [2/2]

bool isCommutative ( const Instruction * I, const Value * ValWithUses, bool IsCopyable = false )

static

Returns

True if I is commutative, handles CmpInst and BinaryOperator. For BinaryOperator, it also checks if InstWithUses is used in specific patterns that make it effectively commutative (like equality comparisons with zero). In most cases, users should not call this function directly (since I and InstWithUses are the same). However, when analyzing interchangeable instructions, we need to use the converted opcode along with the original uses.

Parameters

I	The instruction to check for commutativity
ValWithUses	The value whose uses are analyzed for special patterns

Definition at line 534 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::Value::hasNUsesOrMore(), llvm::Value::hasUseList(), I, llvm::Value::uses(), and UsesLimit.

Referenced by getNumberOfPotentiallyCommutativeOps(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getScoreAtLevelRec(), isCommutableOperand(), isCommutative(), slpvectorizer::BoUpSLP::isProfitableToReorder(), and llvm::FastISel::selectBinaryOp().

isFirstInsertElement()

bool isFirstInsertElement ( const InsertElementInst * IE1, const InsertElementInst * IE2 )

static

Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.

Definition at line 19131 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), getElementIndex(), llvm::Value::hasOneUse(), and llvm_unreachable.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

isFixedVectorShuffle()

std::optional< TargetTransformInfo::ShuffleKind > isFixedVectorShuffle ( ArrayRef< Value * > VL, SmallVectorImpl< int > & Mask, AssumptionCache * AC )

static

Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.

TODO: Can we split off and reuse the shuffle mask detection from ShuffleVectorInst/getShuffleCost?

Definition at line 756 of file SLPVectorizer.cpp.

References llvm::accumulate(), llvm::any_of(), llvm::cast(), llvm::dyn_cast(), E(), llvm::ArrayRef< T >::end(), llvm::find_if(), I, llvm::isa(), llvm::IsaPred, llvm::isGuaranteedNotToBePoison(), isUndefVector(), llvm::PoisonMaskElem, Select, Size, llvm::ArrayRef< T >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, llvm::TargetTransformInfo::SK_PermuteTwoSrc, llvm::TargetTransformInfo::SK_Select, and Unknown.

isMainInstruction()

bool isMainInstruction ( Instruction * I, Instruction * MainOp, Instruction * AltOp, const TargetLibraryInfo & TLI )

static

Checks if the specified instruction I is an main operation for the given MainOp and AltOp instructions.

Definition at line 14038 of file SLPVectorizer.cpp.

References I.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode().

isMaskedLoadCompress() [1/2]

bool isMaskedLoadCompress ( ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo & TTI, const DataLayout & DL, ScalarEvolution & SE, AssumptionCache & AC, const DominatorTree & DT, const TargetLibraryInfo & TLI, const function_ref< bool(Value *)> AreAllUsersVectorized )

static

Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.

Definition at line 7341 of file SLPVectorizer.cpp.

References DL, and isMaskedLoadCompress().

isMaskedLoadCompress() [2/2]

bool isMaskedLoadCompress ( ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo & TTI, const DataLayout & DL, ScalarEvolution & SE, AssumptionCache & AC, const DominatorTree & DT, const TargetLibraryInfo & TLI, const function_ref< bool(Value *)> AreAllUsersVectorized, bool & IsMasked, unsigned & InterleaveFactor, SmallVectorImpl< int > & CompressMask, VectorType *& LoadVecTy )

static

Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.

Definition at line 7200 of file SLPVectorizer.cpp.

References llvm::accumulate(), assert(), llvm::ArrayRef< T >::back(), buildCompressMask(), llvm::CallingConv::C, llvm::cast(), CostKind, DL, llvm::ArrayRef< T >::empty(), llvm::enumerate(), llvm::ArrayRef< T >::front(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::APInt::getAllOnes(), getFullVectorNumberOfElements(), getGEPCosts(), llvm::getPointersDiff(), getScalarizationOverhead(), getShuffleCost(), getWidenedType(), I, InstructionCost, inversePermutation(), llvm::isSafeToLoadUnconditionally(), llvm::PoisonMaskElem, reorderScalars(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::seq(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, llvm::SmallVectorImpl< T >::swap(), and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

isMaskedStoreCompress()

bool isMaskedStoreCompress ( ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo & TTI, const DataLayout & DL, ScalarEvolution & SE, Align CommonAlignment, SmallVectorImpl< int > & ReuseShuffleIndices, FixedVectorType *& StoreVecTy )

static

Checks if the stores VL with pointers PointerOps can be lowered as a single masked store.

On success StoreVecTy is the widened store type and ReuseShuffleIndices is the expand mask that places each stored value at its element offset from the base (poison in the gaps).

Definition at line 7360 of file SLPVectorizer.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::back(), llvm::cast(), llvm::TargetTransformInfo::ConstantMask, DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), llvm::getPointersDiff(), getWidenedType(), I, llvm::Type::isFloatingPointTy(), llvm::Type::isIntOrPtrTy(), llvm::PoisonMaskElem, llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::seq(), and llvm::ArrayRef< T >::size().

isNonVectorizableInst()

bool isNonVectorizableInst ( const Instruction * I, const TargetLibraryInfo * TLI )

static

Returns true if I is an instruction whose result the SLP vectorizer cannot turn into a vector instruction directly, but whose operand chains may still be worth vectorizing as bundle seeds.

Definition at line 32227 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::any_of(), llvm::CallBase::args(), llvm::dyn_cast(), llvm::VFDatabase::getMappings(), getType(), llvm::getVectorIntrinsicIDForCall(), I, II, llvm::isa(), llvm::isTriviallyVectorizable(), and SLPReVec.

isPoorThroughputOp()

bool isPoorThroughputOp ( Instruction * I, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, PoorThroughputOpCache & Cache )

static

Returns true if I is an expensive scalar op whose vector form is cheaper per lane (e.g.

fdiv, frem, fsqrt).

Definition at line 10754 of file SLPVectorizer.cpp.

References llvm::any_of(), CostKind, llvm::dyn_cast(), getPoorThroughputTypeKey(), llvm::getVectorIntrinsicIDForCall(), getVectorOpCost(), I, llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isa(), isValidElementType(), SLPReVec, llvm::TargetTransformInfo::TCC_Expensive, and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

isReductionCandidate()

bool isReductionCandidate ( Instruction * I )

static

\Returns true if I is a candidate instruction for reduction vectorization.

Definition at line 31651 of file SLPVectorizer.cpp.

References I, IsSelect(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and matchRdxBop().

isRepeatedNonIdentityClusteredMask()

bool isRepeatedNonIdentityClusteredMask ( ArrayRef< int > Mask, unsigned Sz )

static

Checks if the given mask is a "clustered" mask with the same clusters of size Sz, which are not identity submasks.

Definition at line 8634 of file SLPVectorizer.cpp.

References E(), I, and llvm::ShuffleVectorInst::isIdentityMask().

isReverseOrder()

bool isReverseOrder ( ArrayRef< unsigned > Order )

static

Check if Order represents reverse order.

Definition at line 6897 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::empty(), llvm::enumerate(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and slpvectorizer::BoUpSLP::getReorderingData().

isSimple()

bool isSimple ( Instruction * I )

static

Returns

True if the instruction is not a volatile or atomic load/store.

Definition at line 1764 of file SLPVectorizer.cpp.

References llvm::dyn_cast(), I, and MI.

Referenced by combineBROADCAST_LOAD(), llvm::DOTGraphTraits< const MachineFunction * >::getNodeLabel(), llvm::DOTGraphTraits< DomTreeNode * >::getNodeLabel(), llvm::DOTGraphTraits< DOTFuncInfo * >::getNodeLabel(), llvm::DOTGraphTraits< DOTMachineFuncInfo * >::getNodeLabel(), llvm::DOTGraphTraits< MachineBlockFrequencyInfo * >::getNodeLabel(), llvm::DOTGraphTraits< RegionNode * >::getNodeLabel(), llvm::X86TargetLowering::shouldReduceLoadWidth(), tryToFoldExtOfMaskedLoad(), llvm::MachineBlockFrequencyInfo::view(), and llvm::MBFIWrapper::view().

isUndefVector()

template<bool IsPoisonOnly = false>

SmallBitVector isUndefVector ( const Value * V, const SmallBitVector & UseMask = {} )

static

Checks if the given value is actually an undefined constant vector.

Also, if the UseMask is not empty, tries to check if the non-masked elements actually mask the insertelement buildvector, if any.

Definition at line 687 of file SLPVectorizer.cpp.

Referenced by generateKeySubkey(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), isFixedVectorShuffle(), and performExtractsShuffleAction().

isUsedOutsideBlock()

bool isUsedOutsideBlock ( Value * V )

static

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all users are phi nodes or instructions from the different blocks.

Definition at line 1905 of file SLPVectorizer.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and UsesLimit.

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

isValidElementType()

bool isValidElementType ( Type * Ty )

static

Predicate for the element types that the SLP vectorizer supports.

The most important thing to filter here are types which are invalid in LLVM vectors. We also filter target specific types which have absolutely no meaningful vectorization path such as x86_fp80 and ppc_f128. This just avoids spending time checking the cost model and realizing that they will be inevitably scalarized.

Definition at line 319 of file SLPVectorizer.cpp.

References llvm::canVectorizeTy(), llvm::getVectorizedTypeVF(), llvm::isVectorizedTy(), SLPReVec, and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::canMapToVector(), compareCmp(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), gatherPossiblyVectorizableLoads(), getFloorFullVectorNumberOfElements(), getFullVectorNumberOfElements(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), hasFullVectorsOrPowerOf2(), isPoorThroughputOp(), tryToVectorizeSequence(), llvm::SLPVectorizerPass::vectorizeCmpInsts(), and llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

isValidForAlternation()

bool isValidForAlternation ( unsigned Opcode )

static

Returns

true if Opcode is allowed as part of the main/alternate instruction for SLP vectorization.

Example of unsupported opcode is SDIV that can potentially cause UB if the "shuffled out" lane would result in division by zero.

Definition at line 862 of file SLPVectorizer.cpp.

References llvm::Instruction::isIntDivRem().

Referenced by generateKeySubkey(), and getSameOpcode().

matchRdxBop()

bool matchRdxBop ( Instruction * I, Value *& V0, Value *& V1 )

static

Definition at line 31598 of file SLPVectorizer.cpp.

References I, llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_FMaximum(), llvm::PatternMatch::m_FMaxNum(), llvm::PatternMatch::m_FMinimum(), llvm::PatternMatch::m_FMinNum(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::V1.

Referenced by getNonPhiOperand(), and isReductionCandidate().

performExtractsShuffleAction()

template<typename T>

T * performExtractsShuffleAction ( MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value * Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action )

static

Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.

It tries to do it in several steps.

1. If the Base vector is not undef vector, resizing the very first mask to have common VF and perform action for 2 input vectors (including non-undef Base). Other shuffle masks are combined with the resulting after the 1 stage and processed as a shuffle of 2 elements.
2. If the Base is undef vector and have only 1 shuffle mask, perform the action only for 1 vector with the given mask, if it is not the identity mask.
3. If > 2 masks are used, perform the remaining shuffle actions for 2 vectors, combing the masks properly between the steps.

Definition at line 19186 of file SLPVectorizer.cpp.

References llvm::SmallBitVector::all(), assert(), llvm::sampleprof::Base, buildUseMask(), E(), I, isUndefVector(), llvm::PoisonMaskElem, T, and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

propagateMetadata()

Instruction * propagateMetadata ( Instruction * Inst, ArrayRef< Value * > VL )

static

Returns

I after propagating metadata from VL only for instructions in VL.

Definition at line 23249 of file SLPVectorizer.cpp.

References llvm::isa(), llvm::propagateMetadata(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

reorderOrder()

void reorderOrder ( SmallVectorImpl< unsigned > & Order, ArrayRef< int > Mask, bool BottomOrder = false )

static

Reorders the given Order according to the given Mask.

Order - is the original order of the scalars. Procedure transforms the provided order in accordance with the given Mask. If the resulting Order is just an identity order, Order is cleared.

Definition at line 6631 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::Data, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), fixupOrderingIndices(), I, inversePermutation(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::PoisonMaskElem, reorderReuses(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

reorderReuses()

void reorderReuses ( SmallVectorImpl< int > & Reuses, ArrayRef< int > Mask )

static

Reorders the given Reuses mask according to the given Mask.

Reuses contains original mask for the scalars reused in the node. Procedure transform this mask in accordance with the given Mask.

Definition at line 6617 of file SLPVectorizer.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), E(), llvm::SmallVectorTemplateCommon< T, typename >::end(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::reorderBottomToTop(), and reorderOrder().

reorderScalars()

void reorderScalars ( SmallVectorImpl< Value * > & Scalars, ArrayRef< int > Mask )

static

Reorders the list of scalars in accordance with the given Mask.

Definition at line 1873 of file SLPVectorizer.cpp.

References assert(), E(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::PoisonValue::get(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::adjustExtracts(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::adjustExtracts(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

replicateMask()

SmallVector< Constant * > replicateMask ( ArrayRef< Constant * > Val, unsigned VF )

static

Replicates the given Val VF times.

Definition at line 1853 of file SLPVectorizer.cpp.

References assert(), llvm::CallingConv::C, llvm::enumerate(), I, llvm::none_of(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather().

shortBundleName()

std::string shortBundleName ( ArrayRef< Value * > VL, int Idx = -1 )

static

Print a short descriptor of the instruction bundle suitable for debug output.

Definition at line 513 of file SLPVectorizer.cpp.

References llvm::ArrayRef< T >::front(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::processBuildVector().

STATISTIC() [1/4]

STATISTIC	(	NumStoreChains	,
		"Number of vector stores created"	)

STATISTIC() [2/4]

STATISTIC	(	NumStridedStoreChains	,
		"Number of vectorized stride stores"	)

STATISTIC() [3/4]

STATISTIC	(	NumVectorInstructions	,
		"Number of vector instructions generated"	)

STATISTIC() [4/4]

STATISTIC	(	NumVectorizedStores	,
		"Number of vectorized stores"	)

transformScalarShuffleIndiciesToVector()

void transformScalarShuffleIndiciesToVector ( unsigned VecTyNumElements, SmallVectorImpl< int > & Mask )

static

Definition at line 403 of file SLPVectorizer.cpp.

References llvm::enumerate(), I, llvm::MutableArrayRef(), llvm::PoisonMaskElem, and llvm::seq().

tryGetSecondaryReductionRoot()

Instruction * tryGetSecondaryReductionRoot ( PHINode * Phi, Instruction * Root )

static

We could have an initial reduction that is not an add.

r *= v1 + v2 + v3 + v4 In such a case start looking for a tree rooted in the first '+'. \Returns the new root if found, which may be nullptr if not an instruction.

Definition at line 31624 of file SLPVectorizer.cpp.

References assert(), llvm::dyn_cast(), llvm::User::getOperand(), llvm::isa(), LHS, and RHS.

tryToFindDuplicates()

bool tryToFindDuplicates ( SmallVectorImpl< Value * > & VL, SmallVectorImpl< int > & ReuseShuffleIndices, const TargetTransformInfo & TTI, const TargetLibraryInfo & TLI, const InstructionsState & S, const BoUpSLP::EdgeInfo & UserTreeIdx, const BoUpSLP & R, bool BuildGatherOnly = true )

static

Checks that every instruction appears once in the list and if not, packs them, building ReuseShuffleIndices mask and mutating VL.

The list of unique scalars is extended by poison values to the whole register size.

Returns

false if VL could not be uniquified, in which case VL is unchanged and ReuseShuffleIndices is empty.

Definition at line 11515 of file SLPVectorizer.cpp.

References llvm::all_of(), assert(), llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::APInt::clearBit(), slpvectorizer::BoUpSLP::CompressVectorize, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), CostKind, llvm::CostThreshold, llvm::dbgs(), doesNotNeedToSchedule(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), llvm::SmallVectorImpl< T >::erase(), llvm::find_if_not(), llvm::SmallVectorTemplateCommon< T, typename >::front(), slpvectorizer::BoUpSLP::Gather, llvm::APInt::getAllOnes(), getNumberOfParts(), getScalarizationOverhead(), getShuffleCost(), getValueType(), getWidenedType(), llvm::APInt::getZero(), llvm::has_single_bit(), InstructionCost, llvm::isa(), llvm::IsaPred, isConstant(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::slpvectorizer::isVectorLikeInstWithConstOps(), LLVM_DEBUG, llvm::make_range(), llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::rbegin(), llvm::SmallVectorTemplateCommon< T, typename >::rend(), slpvectorizer::BoUpSLP::ScatterVectorize, llvm::APInt::setBit(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, slpvectorizer::BoUpSLP::StridedVectorize, llvm::TargetTransformInfo::TCC_Expensive, llvm::TargetTransformInfo::TCC_Free, llvm::TargetTransformInfo::TCK_RecipThroughput, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace(), and slpvectorizer::BoUpSLP::EdgeInfo::UserTE.

tryToVectorizeSequence()

template<typename T>

bool tryToVectorizeSequence ( SmallVectorImpl< T * > & Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(ArrayRef< T * >, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP & R )

static

\Returns the minimum number of elements that we will attempt to vectorize.

Definition at line 31968 of file SLPVectorizer.cpp.

References llvm::ArrayRef(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), Changed, llvm::SmallVectorImpl< T >::clear(), llvm::dbgs(), llvm::dyn_cast(), E(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorTemplateCommon< T, typename >::front(), getValueType(), I, llvm::isa(), isValidElementType(), LLVM_DEBUG, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::stable_sort(), llvm::SmallVectorImpl< T >::swap(), and T.

Referenced by llvm::SLPVectorizerPass::vectorizeCmpInsts(), and llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

Variable Documentation

AliasedCheckLimit

const unsigned AliasedCheckLimit = 10

static

Definition at line 294 of file SLPVectorizer.cpp.

DisableTreeReorder

cl::opt< bool > DisableTreeReorder("slp-disable-tree-reorder", cl::init(false), cl::Hidden, cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only.")) ( "slp-disable-tree-reorder" , cl::init(false) , cl::Hidden , cl::desc("Disable tree reordering even if it is " "profitable. Used for testing only.")  )

static

Referenced by slpvectorizer::BoUpSLP::isProfitableToReorder().

EnableMaskedStores

cl::opt< bool > EnableMaskedStores("slp-enable-masked-stores", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores.")) ( "slp-enable-masked-stores" , cl::init(true) , cl::Hidden , cl::desc("Enable vectorization of non-consecutive stores as a single " "masked store, when the target supports masked stores.")  )

static

EnableStridedStores

cl::opt< bool > EnableStridedStores("slp-enable-strided-stores", cl::init(false), cl::Hidden, cl::desc("Enable SLP trees to be built from strided " "store chains.")) ( "slp-enable-strided-stores" , cl::init(false) , cl::Hidden , cl::desc("Enable SLP trees to be built from strided " "store chains.")  )

static

ForcePostProcessStoresOperands

cl::opt< bool > ForcePostProcessStoresOperands("slp-postprocess-stores-operands", cl::init(false), cl::Hidden, cl::desc("Force vectorization of non-vectorizable stores operands.")) ( "slp-postprocess-stores-operands" , cl::init(false) , cl::Hidden , cl::desc("Force vectorization of non-vectorizable stores operands.")  )

static

ForceStridedLoads

cl::opt< bool > ForceStridedLoads("slp-force-strided-loads", cl::init(false), cl::Hidden, cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only.")) ( "slp-force-strided-loads" , cl::init(false) , cl::Hidden , cl::desc("Generate strided loads even if they are not " "profitable. Used for testing only.")  )

static

LookAheadMaxDepth

cl::opt< int > LookAheadMaxDepth("slp-max-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for operand reordering scores")) ( "slp-max-look-ahead-depth" , cl::init(2) , cl::Hidden , cl::desc("The maximum look-ahead depth for operand reordering scores")  )

static

LoopAwareTripCount

cl::opt< unsigned > LoopAwareTripCount("slp-cost-loop-trip-count", cl::init(2), cl::Hidden, cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)")) ( "slp-cost-loop-trip-count" , cl::init(2) , cl::Hidden , cl::desc("Loop trip count, considered by the cost model during " "modeling (0=loops are ignored and considered flat code)")  )

static

Referenced by getLoopTripCount().

MaxMemDepDistance

const unsigned MaxMemDepDistance = 160

static

Definition at line 303 of file SLPVectorizer.cpp.

MaxPHINumOperands

const unsigned MaxPHINumOperands = 128

static

Maximum allowed number of operands in the PHI nodes.

Definition at line 310 of file SLPVectorizer.cpp.

MaxProfitableStride

cl::opt< unsigned > MaxProfitableStride("slp-max-stride", cl::init(8), cl::Hidden, cl::desc("The maximum stride, considered to be profitable.")) ( "slp-max-stride" , cl::init(8) , cl::Hidden , cl::desc("The maximum stride, considered to be profitable.")  )

static

Referenced by slpvectorizer::BoUpSLP::isStridedLoad().

MaxVectorRegSizeOption

cl::opt< int > MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) ( "slp-max-reg-size" , cl::init(128) , cl::Hidden , cl::desc("Attempt to vectorize for this register size in bits")  )

static

Referenced by slpvectorizer::BoUpSLP::BoUpSLP().

MaxVFOption

cl::opt< unsigned > MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden, cl::desc("Maximum SLP vectorization factor (0=unlimited)")) ( "slp-max-vf" , cl::init(0) , cl::Hidden , cl::desc("Maximum SLP vectorization factor (0=unlimited)")  )

static

Referenced by slpvectorizer::BoUpSLP::getMaximumVF().

MinProfitableStridedLoads

cl::opt< unsigned > MinProfitableStridedLoads("slp-min-strided-loads", cl::init(2), cl::Hidden, cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value")) ( "slp-min-strided-loads" , cl::init(2) , cl::Hidden , cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value")  )

static

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate(), and slpvectorizer::BoUpSLP::isStridedLoad().

MinProfitableStridedStores

cl::opt< unsigned > MinProfitableStridedStores("slp-min-strided-stores", cl::init(2), cl::Hidden, cl::desc( "The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value")) ( "slp-min-strided-stores" , cl::init(2) , cl::Hidden , cl::desc( "The minimum number of stores, which should be considered strided, " "if the stride is > 1 or is runtime value")  )

static

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate().

MinScheduleRegionSize

const int MinScheduleRegionSize = 16

static

If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

Definition at line 307 of file SLPVectorizer.cpp.

MinTreeSize

cl::opt< unsigned > MinTreeSize("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable")) ( "slp-min-tree-size" , cl::init(3) , cl::Hidden , cl::desc("Only vectorize small trees if they are fully vectorizable")  )

static

Referenced by slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

MinVectorRegSizeOption

cl::opt< int > MinVectorRegSizeOption("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits")) ( "slp-min-reg-size" , cl::init(128) , cl::Hidden , cl::desc("Attempt to vectorize for this register size in bits")  )

static

Referenced by slpvectorizer::BoUpSLP::BoUpSLP().

NonVectReductions

cl::opt< bool > NonVectReductions("slp-non-vectorizables-as-reductions", cl::init(false), cl::Hidden, cl::desc( "Use non-vectorizable instructions as potential reduction roots.")) ( "slp-non-vectorizables-as-reductions" , cl::init(false) , cl::Hidden , cl::desc( "Use non-vectorizable instructions as potential reduction roots.")  )

static

Referenced by forEachOperandChainCandidate().

PerLaneGatherScale

cl::opt< bool > PerLaneGatherScale("slp-per-lane-gather-scale", cl::init(true), cl::Hidden, cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences.")) ( "slp-per-lane-gather-scale" , cl::init(true) , cl::Hidden , cl::desc("Use per-lane execution scale for gather/buildvector tree " "entries to model LICM-hoistable buildvector sequences.")  )

static

Refine the loop-aware cost scaling of gather/buildvector tree entries by using the per-lane execution scale of the operand that feeds each lane, instead of a single whole-entry scale.

This matches the LICM hoisting performed by optimizeGatherSequence() at codegen time: lanes whose operands are loop-invariant in an inner loop contribute the outer loop's execution scale rather than the inner loop's, which avoids over-costing buildvectors that bridge values from outer loop nests into an inner loop.

PHINodeVectorizationBudget

cl::opt< unsigned > PHINodeVectorizationBudget("slp-phi-vectorization-budget", cl::init(1024), cl::Hidden, cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs")) ( "slp-phi-vectorization-budget" , cl::init(1024) , cl::Hidden , cl::desc("Do not vectorize a bundle of PHI nodes if the product of the " "bundle size and the number of incoming values exceeds this " "value, to limit the compile time spent on wide PHIs")  )

static

RecursionMaxDepth

cl::opt< unsigned > RecursionMaxDepth("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree")) ( "slp-recursion-max-depth" , cl::init(12) , cl::Hidden , cl::desc("Limit the recursion depth when building a vectorizable tree")  )

static

Referenced by arePointersCompatible(), clusterSortPtrAccesses(), gatherPossiblyVectorizableLoads(), slpvectorizer::BoUpSLP::getVectorElementSize(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

RootLookAheadMaxDepth

cl::opt< int > RootLookAheadMaxDepth("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for searching best rooting option")) ( "slp-max-root-look-ahead-depth" , cl::init(2) , cl::Hidden , cl::desc("The maximum look-ahead depth for searching best rooting option")  )

static

Referenced by slpvectorizer::BoUpSLP::findBestRootPair().

RunSLPVectorization

cl::opt< bool > RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden, cl::desc("Run the SLP vectorization passes")) ( "vectorize-slp" , cl::init(true) , cl::Hidden , cl::desc("Run the SLP vectorization passes")  )

static

Referenced by llvm::SLPVectorizerPass::runImpl().

ScheduleRegionSizeBudget

cl::opt< int > ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block")) ( "slp-schedule-budget" , cl::init(100000) , cl::Hidden , cl::desc("Limit the size of the SLP scheduling region per block")  )

static

Limits the size of scheduling regions in a block.

It avoid long compile times for very large blocks where vector instructions are spread over a wide range. This limit is way higher than needed by real-world functions.

Referenced by slpvectorizer::BoUpSLP::getSpillCost().

ShouldStartVectorizeHorAtStore

cl::opt< bool > ShouldStartVectorizeHorAtStore("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc( "Attempt to vectorize horizontal reductions feeding into a store")) ( "slp-vectorize-hor-store" , cl::init(false) , cl::Hidden , cl::desc( "Attempt to vectorize horizontal reductions feeding into a store")  )

static

ShouldVectorizeHor

cl::opt< bool > ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions")) ( "slp-vectorize-hor" , cl::init(true) , cl::Hidden , cl::desc("Attempt to vectorize horizontal reductions")  )

static

SLPCostThreshold

cl::opt< int > SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this " "number ")) ( "slp-threshold" , cl::init(0) , cl::Hidden , cl::desc("Only vectorize if you gain more than this " "number ")  )

static

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), slpvectorizer::BoUpSLP::getTreeCost(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

SLPInstCountCheck

cl::opt< bool > SLPInstCountCheck("slp-inst-count-check", cl::init(true), cl::Hidden, cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count")) ( "slp-inst-count-check" , cl::init(true) , cl::Hidden , cl::desc("Reject vectorization if vector instruction count exceeds " "scalar instruction count")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

SLPReVec

cl::opt< bool > SLPReVec("slp-revec", cl::init(false), cl::Hidden, cl::desc("Enable vectorization for wider vector utilization")) ( "slp-revec" , cl::init(false) , cl::Hidden , cl::desc("Enable vectorization for wider vector utilization")  )

static

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather(), getExtractWithExtendCost(), slpvectorizer::BoUpSLP::getTreeCost(), getValueType(), getVectorInstrCost(), isNonVectorizableInst(), isPoorThroughputOp(), slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable(), isValidElementType(), slpvectorizer::BoUpSLP::reorderTopToBottom(), and slpvectorizer::BoUpSLP::vectorizeTree().

SplitAlternateInstructions

cl::opt< bool > SplitAlternateInstructions("slp-split-alternate-instructions", cl::init(true), cl::Hidden, cl::desc("Improve the code quality by splitting alternate instructions")) ( "slp-split-alternate-instructions" , cl::init(true) , cl::Hidden , cl::desc("Improve the code quality by splitting alternate instructions")  )

static

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode().

UsesLimit

int UsesLimit = 64

staticconstexpr

Definition at line 298 of file SLPVectorizer.cpp.

Referenced by slpvectorizer::BoUpSLP::buildExternalUses(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getTreeCost(), isCommutative(), and isUsedOutsideBlock().

VectorizeCopyableElements

cl::opt< bool > VectorizeCopyableElements("slp-copyable-elements", cl::init(true), cl::Hidden, cl::desc("Try to replace values with the idempotent instructions for " "better vectorization.")) ( "slp-copyable-elements" , cl::init(true) , cl::Hidden , cl::desc("Try to replace values with the idempotent instructions for " "better vectorization.")  )

static

Enables vectorization of copyable elements.

VectorizeNonPowerOf2

cl::opt< bool > VectorizeNonPowerOf2("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, cl::desc("Try to vectorize with non-power-of-2 number of elements.")) ( "slp-vectorize-non-power-of-2" , cl::init(false) , cl::Hidden , cl::desc("Try to vectorize with non-power-of-2 number of elements.")  )

static

Referenced by isAllowedNonPowerOf2VF().

VectorizePoorThroughput

cl::opt< bool > VectorizePoorThroughput("slp-vectorize-poor-throughput", cl::init(true), cl::Hidden, cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds.")) ( "slp-vectorize-poor-throughput" , cl::init(true) , cl::Hidden , cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as " "standalone vectorization seeds.")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

ViewSLPTree

cl::opt< bool > ViewSLPTree("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz")) ( "view-slp-tree" , cl::Hidden , cl::desc("Display the SLP trees with Graphviz")  )

static

Referenced by slpvectorizer::BoUpSLP::getTreeCost().