A private "module" namespace for types and utilities used by this pass. More...

Classes
class	BaseShuffleAnalysis
	The base class for shuffle instruction emission and shuffle cost estimation. More...
class	BinOpSameOpcodeHelper
	Helper class that determines VL can use the same opcode. More...
struct	BitPackInfo
	Description of a bitfield packing of vector lanes into a scalar value: every lane contributes a disjoint contiguous byte field of the result. More...
class	BoUpSLP
	Bottom Up SLP Vectorizer. More...
class	CmpSamePredicateHelper
	Helper class that determines whether a list of integer comparisons can share a single predicate. More...
class	InstructionsState
	Main data required for vectorization of instructions. More...
struct	NarrowedLeafInfo
	Narrow reduction leaf: the value, the shift applied after widening and the mask applied in the narrow type before widening, clearing the bits the absorbed narrow shls shift out and applying the absorbed narrow and-masks. More...

Enumerations
enum class	BoolBitmask { None , NoMask , NeedMask }
	The result of matching a boolean bitmask reduction over narrowed leaves. More...
enum class	UseMask { FirstArg , SecondArg , UndefsAsMask }
	Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask. More...

Functions
bool	isValidForAlternation (unsigned Opcode)
static std::optional< std::pair< bool, APInt > >	getCmpBoundaryFamily (CmpInst::Predicate Pred, const APInt &C)
	If the comparison (Pred, X, C) is a single-element or single-complement range check, returns its boundary family: false + K for the singleton {K} (eq forms), true + K for the complement of {K} (ne forms).
bool	isAbsorbableFMulOrFAdd (ArrayRef< Value * > VL, Value *V)
	Checks if `V` is a single-use fmul/fadd with operands outside `VL`.
bool	isAbsorbableCopyableFMulOrFAdd (const InstructionsState &S, Value *V)
	Checks if `V` is a copyable single-use fmul/fadd, absorbable as fmuladd(a, b, -0.0) or fmuladd(1.0, a, b).
bool	hasOnlyAbsorbableCopyableFMulOrFAdds (ArrayRef< Value * > VL)
	Checks if every copyable in `VL` is an absorbable fmul/fadd: the binops die instead of being computed and gathered.
static Instruction *	findInstructionWithOpcode (ArrayRef< Value * > VL, unsigned Opcode)
	Find an instruction with a specific opcode in VL.
static bool	areCompatibleCmpOps (Value BaseOp0, Value BaseOp1, Value Op0, Value Op1, const TargetLibraryInfo &TLI)
	Checks if the provided operands of 2 cmp instructions are compatible, i.e.
static bool	isCmpSameOrSwapped (const CmpInst BaseCI, const CmpInst CI, const TargetLibraryInfo &TLI)
InstructionsState	getSameOpcode (ArrayRef< Value * > VL, const TargetLibraryInfo &TLI)
std::pair< Instruction , SmallVector< Value > >	convertTo (Instruction *I, const InstructionsState &S)
bool	isAlternateInstruction (Instruction I, Instruction MainOp, Instruction *AltOp, const TargetLibraryInfo &TLI)
	Checks if the specified instruction `I` is an alternate operation for the given `MainOp` and `AltOp` instructions.
SmallVector< SmallVector< Value * > >	scanAltAssociativeOperands (const InstructionsState &S, const TargetLibraryInfo &TLI, ArrayRef< Value * > VL, ArrayRef< Value * > Op0, ArrayRef< Value * > Op1, SmallVectorImpl< Value * > &ReassocScalars, SmallBitVector &SubLanes)
	Peel the per-lane associative chains of an alternate node into operand columns.
InstructionCost	getShuffleCost (const TargetTransformInfo &TTI, TargetTransformInfo::ShuffleKind Kind, VectorType Tp, const TargetTransformInfo::TargetCostKind CostKind, ArrayRef< int > Mask={}, int Index=0, VectorType SubTp=nullptr, ArrayRef< const Value * > Args={}, TargetTransformInfo::VectorInstrContext VIC=TargetTransformInfo::VectorInstrContext::None)
	Returns the cost of the shuffle instructions with the given `Kind`, vector type `Tp` and optional `Mask`.
std::pair< InstructionCost, InstructionCost >	getGEPCosts (const TargetTransformInfo &TTI, ArrayRef< Value * > Ptrs, Value BasePtr, unsigned Opcode, const TargetTransformInfo::TargetCostKind CostKind, Type ScalarTy, VectorType *VecTy)
	Calculate the scalar and the vector costs from vectorizing set of GEPs.
InstructionCost	getBlendedLoadCost (const TargetTransformInfo &TTI, Type *VecTy, Align Alignment, unsigned AddressSpace, const TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of a BlendedLoadVectorize node loading `VecTy:` two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.
InstructionCost	getWidenedStridedCastCost (const TargetTransformInfo &TTI, Type SrcTy, Type DstTy, const DataLayout &DL, TargetTransformInfo::CastContextHint CCH, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of the cast between the widened strided access type and the entry vector type.
InstructionCost	getMaskedDivRemCost (const TargetTransformInfo &TTI, bool ReVec, unsigned Opcode, Type ScalarTy, unsigned NumElts, const TargetTransformInfo::TargetCostKind CostKind, FixedVectorType *PaddedTy=nullptr)
	For a non-power-of-2 `NumElts-wide` integer div/rem `Opcode`, checks if padding to a full register and using the masked div/rem intrinsic is cheaper than the direct vector op.
InstructionCost	getScalarizationOverhead (const TargetTransformInfo &TTI, bool ReVec, Type ScalarTy, VectorType Ty, const APInt &DemandedElts, bool Insert, bool Extract, const TargetTransformInfo::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TargetTransformInfo::VectorInstrContext VIC=TargetTransformInfo::VectorInstrContext::None)
	This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorType, a vector will be inserted or extracted instead of a scalar.
InstructionCost	getVectorInstrCost (const TargetTransformInfo &TTI, bool ReVec, Type ScalarTy, unsigned Opcode, Type Val, const TargetTransformInfo::TargetCostKind CostKind, unsigned Index, Value Scalar, ArrayRef< std::tuple< Value , User *, int > > ScalarUserAndIdx, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None)
	This is similar to TargetTransformInfo::getVectorInstrCost, but if ScalarTy is a FixedVectorType, a vector will be extracted instead of a scalar.
InstructionCost	getExtractWithExtendCost (const TargetTransformInfo &TTI, bool ReVec, unsigned Opcode, Type Dst, VectorType VecTy, unsigned Index, const TargetTransformInfo::TargetCostKind CostKind)
	This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst is a FixedVectorType, a vector will be extracted instead of a scalar.
static TTI::CastContextHint	getBoolReduxResultCCH (const Value *Root)
	Returns the cast context hint for the trunc of the booleanized reduction result, which inherits the uses of the reduction root `Root`.
InstructionCost	getBoolReduxWideRdxCost (const TargetTransformInfo &TTI, RecurKind RdxKind, FixedVectorType VecTy, const Value Root, FastMathFlags FMF, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of the booleanized logical and/or reduction of a vector of type `VecTy` with the i1 root `Root`, emitted as the wide reduction plus the result trunc.
InstructionCost	getBoolReduxBitcastCmpCost (const TargetTransformInfo &TTI, RecurKind RdxKind, FixedVectorType VecTy, const Value Root, ArrayRef< Instruction * > ChainInsts, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of the booleanized logical and/or reduction of a vector of type `VecTy` with the i1 root `Root`, emitted as trunc+bitcast+cmp, estimated in the context of the replaced cast chain `ChainInsts`.
static InstructionCost	getBoolLogicRdxBitcastCost (RecurKind Kind, const TargetTransformInfo &TTI, FixedVectorType *VectorTy, TTI::CastContextHint Ctx, TTI::TargetCostKind CostKind)
std::pair< InstructionCost, bool >	getI1ReductionCost (RecurKind Kind, const TargetTransformInfo &TTI, FixedVectorType VectorTy, Type ScalarTy, TargetTransformInfo::CastContextHint Ctx, TargetTransformInfo::TargetCostKind CostKind)
	i1 reductions can be emitted as the plain target reduction or in the bitcast-based form (bitcast to a scalar integer type plus a compare for and/or, plus ctpop for add).
InstructionCost	getBitPackCost (const TargetTransformInfo &TTI, FixedVectorType SrcTy, Type ResultTy, const BitPackInfo &Info, unsigned ZExtSrcWidth, TargetTransformInfo::CastContextHint CCH, TargetTransformInfo::TargetCostKind CostKind, const TargetLibraryInfo TLI, const Instruction CtxI, unsigned &ShiftWidth)
	Returns the cost of the bitfield packing of `SrcTy` into `ResultTy`, picking the cheapest shift width.
InstructionCost	getBoolBitmaskCost (const TargetTransformInfo &TTI, bool NeedMask, Type NarrowScalarTy, Type WideTy, unsigned VF, ArrayRef< int > PermMask, const Value *Root, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of the boolean bitmask reduction of a vector of boolean leaves of type `NarrowScalarTy`, emitted as [and] + [lane permutation `PermMask`] + zero test + bitcast [+ zext] to `WideTy`.
InstructionCost	getNarrowedLeafOpsCost (const TargetTransformInfo &TTI, const SmallDenseMap< Value , NarrowedLeafInfo > &NarrowedLeafShifts, VectorType NarrowVecTy, VectorType WideVecTy, const Instruction CtxI, TargetTransformInfo::TargetCostKind CostKind)
	Returns the cost of the per-lane operations on the narrowed leaves `NarrowedLeafShifts:` the shl in the wide vector type if any leaf is shifted and the and in the narrow vector type if any leaf is masked.
Value *	createWidenedStridedCast (IRBuilderBase &Builder, Value V, Type DstTy, const DataLayout &DL)
	Casts `V` between the widened strided access type and the entry vector type `DstTy`.
ConstantInt *	getStrideBytesIfConstant (Value Stride, Type ScalarTy, const DataLayout &DL, bool IsReverse=false)
	Returns `Stride` scaled by the allocation size of `ScalarTy`, negated if `IsReverse` is set, or nullptr if `Stride` is not a constant.
bool	arePointersCompatible (Value Ptr1, Value Ptr2, const TargetLibraryInfo &TLI, unsigned MaxDepth, bool CompareOpcodes=true)
	`MaxDepth` is the recursion limit for getUnderlyingObject.
template<typename T>
Align	computeCommonAlignment (ArrayRef< Value * > VL)
	Calculates minimal alignment as a common alignment.
template Align	computeCommonAlignment< LoadInst > (ArrayRef< Value * >)
template Align	computeCommonAlignment< StoreInst > (ArrayRef< Value * >)
const SCEV *	calculateRtStride (ArrayRef< Value * > PointerOps, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
	Checks if the provided list of pointers `Pointers` represents the strided pointers for type ElemTy.
static bool	buildCompressMask (ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, Type *ScalarTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< int > &CompressMask)
	Builds compress-like mask for shuffles for the given `PointerOps`, ordered with `Order`.
bool	isMaskedLoadCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, AssumptionCache &AC, const DominatorTree &DT, const TargetLibraryInfo &TLI, const TargetTransformInfo::TargetCostKind CostKind, const function_ref< bool(Value )> AreAllUsersVectorized, bool ReVec, bool &IsMasked, unsigned &InterleaveFactor, SmallVectorImpl< int > &CompressMask, VectorType &LoadVecTy)
	Checks if the `VL` can be transformed to a (masked)load + compress or (masked) interleaved load.
bool	isMaskedLoadCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, AssumptionCache &AC, const DominatorTree &DT, const TargetLibraryInfo &TLI, const TargetTransformInfo::TargetCostKind CostKind, const function_ref< bool(Value *)> AreAllUsersVectorized, bool ReVec)
	Checks if the `VL` can be transformed to a (masked)load + compress or (masked) interleaved load.
bool	canWidenLoadsOverUndefLanes (ArrayRef< Value * > VL, const DataLayout &DL, ScalarEvolution &SE, AssumptionCache &AC, const DominatorTree &DT, const TargetLibraryInfo &TLI)
	Checks if the simple loads of `VL`, mixed with the undef and poison lanes, can be replaced by a single load of the whole `VL` width.
uint64_t	getVectorLoadShift (ArrayRef< Value * > Scalars)
	Returns the distance in bytes from the start of the vector load to the first load of `Scalars:` the leading undef lanes are covered by the vector load.
bool	isMaskedStoreCompress (ArrayRef< Value * > VL, ArrayRef< Value * > PointerOps, ArrayRef< unsigned > Order, const TargetTransformInfo &TTI, const DataLayout &DL, ScalarEvolution &SE, Align CommonAlignment, SmallVectorImpl< int > &ReuseShuffleIndices, FixedVectorType *&StoreVecTy)
	Checks if the stores `VL` with pointers `PointerOps` can be lowered as a single masked store.
bool	clusterSortPtrAccesses (ArrayRef< Value * > VL, ArrayRef< BasicBlock * > BBs, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, unsigned MaxDepth, SmallVectorImpl< unsigned > &SortedIndices)
	Clusters `VL` pointers by (basic block, underlying object) pair and sorts each cluster by offset.
static bool	matchRdxBop (Instruction I, Value &V0, Value *&V1)
Instruction *	getNonPhiOperand (Instruction I, PHINode Phi)
bool	isReductionCandidate (Instruction *I)
Type *	getBoolReduxWideTy (RecurKind RdxKind, Type RootTy, Type LeafTy)
BoolBitmask	isBoolBitmaskRdx (RecurKind RdxKind, const SmallDenseMap< Value *, NarrowedLeafInfo > &NarrowedLeafShifts, const DataLayout &DL)
bool	matchPackedFields (Value V, unsigned MaxDepth, SmallVectorImpl< Value > &Fields, SmallVectorImpl< Instruction * > &Chain)
	Matches `V` as the zero-extended fields, placed one after another from the lowest bit and filling `V` completely.
Value *	tryEmitBoolReduxBitcastCmp (IRBuilderBase &Builder, const TargetTransformInfo &TTI, RecurKind RdxKind, Value Vec, const Value Root, FastMathFlags FMF, TargetTransformInfo::TargetCostKind CostKind)
	Emits the booleanized logical and/or reduction of `Vec` with the i1 root `Root` as trunc+bitcast+cmp (all-ones comparison for And, zero for Or) if it is cheaper than the wide reduction plus the result trunc.
bool	isValidElementType (Type *Ty, bool ReVec)
	Predicate for the element types that the SLP vectorizer supports.
Type *	getValueType (Value *V, bool ReVec, bool LookThroughCmp=false)
	Returns the "element type" of the given value/instruction `V`.
Type *	getWidenedType (Type *ScalarTy, unsigned VF)
unsigned	getFullVectorNumberOfElements (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz, bool ReVec)
	Returns the number of elements of the given type `Ty`, not less than `Sz`, which forms type, which splits by `TTI` into whole vector types during legalization.
unsigned	getFloorFullVectorNumberOfElements (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz, bool ReVec)
	Returns the number of elements of the given type `Ty`, not greater than `Sz`, which forms type, which splits by `TTI` into whole vector types during legalization.
FixedVectorType *	getMaskedDivRemType (const TargetTransformInfo &TTI, unsigned Opcode, Type *ScalarTy, unsigned NumElts, bool ReVec)
	For a non-power-of-2 `NumElts-wide` integer div/rem `Opcode`, returns the padded full-register vector type if padding is structurally possible, or nullptr if the vector already fills a register or the opcode is not div/rem.
bool	hasFullVectorsOrPowerOf2 (const TargetTransformInfo &TTI, Type *Ty, unsigned Sz, bool ReVec)
	Returns true if widened type of `Ty` elements with size `Sz` represents full vector type, i.e.
bool	isAllowedNonPowerOf2VF (unsigned NumElts, bool AllowNonPowerOf2)
	True when `AllowNonPowerOf2` is set and `NumElts` is a supported non-power-of-2 width: `NumElts` + 1 must be a power of two (e.g.
unsigned	getNumberOfPartsOrRegs (bool QueryNumParts, const TargetTransformInfo &TTI, Type VecTy, Type ScalarTy, bool ReVec, unsigned Limit=std::numeric_limits< unsigned >::max())
	Returns number of parts, the type `VecTy` will be split at the codegen phase.
bool	isTypeLimitNeutral (const Value *V, bool IsMax, const DataLayout &DL)
	Returns true if the type limit of the smax (`IsMax`) or smin lane `V` stays neutral after narrowing.
bool	isConstant (Value *V)
bool	isBinOpIdentityConstant (const Value *V, unsigned Opcode)
bool	isUndefTolerantBinOp (const Value *V)
const LoadInst *	getLoadOfUndefTolerantBinOp (const Value *V)
unsigned	getReassocCombineOpcode (unsigned Opcode)
bool	isReassocChainLink (const Instruction *I)
bool	isVectorLikeInstWithConstOps (Value *V)
	Checks if `V` is one of vector-like instructions, i.e.
unsigned	getNumElements (Type *Ty)
unsigned	getPartNumElems (unsigned Size, unsigned NumParts)
	Returns power-of-2 number of elements in a single register (part), given the total number of elements `Size` and number of registers (parts) `NumParts`.
unsigned	getNumElems (unsigned Size, unsigned PartNumElems, unsigned Part)
	Returns correct remaining number of elements, considering total amount `Size`, (power-of-2 number) of elements in a single register `PartNumElems` and current register (part) `Part`.
std::string	shortBundleName (ArrayRef< Value * > VL, int Idx=-1)
	Print a short descriptor of the instruction bundle suitable for debug output.
bool	allSameBlock (ArrayRef< Value * > VL)
bool	allConstant (ArrayRef< Value * > VL)
bool	isSplat (ArrayRef< Value * > VL)
Intrinsic::ID	isEquivalentIntrinsicID (Intrinsic::ID LHS, Intrinsic::ID RHS)
	Checks if `LHS` and `RHS` are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.
bool	isCommutative (const Instruction I, const Value ValWithUses, bool IsCopyable)
bool	isCommutative (const Instruction *I)
	This is a helper function to check whether `I` is commutative.
bool	isCommutableOperand (const Instruction I, Value ValWithUses, unsigned Op, bool IsCopyable=false)
	Checks if the operand is commutative.
unsigned	getNumberOfPotentiallyCommutativeOps (Instruction *I)
std::optional< unsigned >	getElementIndex (const Value *Inst, unsigned Offset)
bool	allSameOpcode (ArrayRef< Value * > VL)
std::optional< unsigned >	getExtractIndex (const Instruction *E)
void	inversePermutation (ArrayRef< unsigned > Indices, SmallVectorImpl< int > &Mask)
	Compute the inverse permutation `Mask` of `Indices`.
void	reorderScalars (SmallVectorImpl< Value * > &Scalars, ArrayRef< int > Mask)
	Reorders the list of scalars in accordance with the given `Mask`.
void	reorderReuses (SmallVectorImpl< int > &Reuses, ArrayRef< int > Mask)
	Reorders the given `Reuses` mask according to the given `Mask`.
void	reorderOrder (SmallVectorImpl< unsigned > &Order, ArrayRef< int > Mask, bool BottomOrder=false)
	Reorders the given `Order` according to the given `Mask`.
bool	isReverseOrder (ArrayRef< unsigned > Order)
	Check if `Order` represents reverse order.
bool	isRepeatedNonIdentityClusteredMask (ArrayRef< int > Mask, unsigned Sz)
	Checks if the given mask is a "clustered" mask with the same clusters of size `Sz`, which are not identity submasks.
void	combineOrders (MutableArrayRef< unsigned > Order, ArrayRef< unsigned > SecondaryOrder)
	Fills unset elements of `Order` (marked with the sentinel value equal to the order size) with the corresponding elements of `SecondaryOrder`, skipping already used indices, or with the identity order if `SecondaryOrder` is empty.
bool	allSameType (ArrayRef< Value * > VL)
template<typename T>
std::optional< unsigned >	getInsertExtractIndex (const Value *Inst, unsigned Offset)
template std::optional< unsigned >	getInsertExtractIndex< InsertElementInst > (const Value *, unsigned)
template std::optional< unsigned >	getInsertExtractIndex< ExtractElementInst > (const Value *, unsigned)
bool	areAllOperandsNonInsts (Value *V)
	Checks if the provided value does not require scheduling.
bool	isUsedOutsideBlock (Value *V)
	Checks if the provided value does not require scheduling.
bool	doesNotNeedToBeScheduled (Value *V)
	Checks if the specified value does not require scheduling.
bool	doesNotNeedToSchedule (ArrayRef< Value * > VL)
	Checks if the specified array of instructions does not require scheduling.
void	transformScalarShuffleIndiciesToVector (unsigned VecTyNumElements, SmallVectorImpl< int > &Mask)
unsigned	getShufflevectorNumGroups (ArrayRef< Value * > VL)
SmallVector< int >	calculateShufflevectorMask (ArrayRef< Value * > VL)
std::optional< TargetTransformInfo::ShuffleKind >	isFixedVectorShuffle (ArrayRef< Value * > VL, SmallVectorImpl< int > &Mask, AssumptionCache *AC)
	Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.
Value *	createInsertVector (IRBuilderBase &Builder, Value Vec, Value V, unsigned Index, function_ref< Value (Value , Value *, ArrayRef< int >)> Generator={})
	Creates subvector insert.
Value *	createExtractVector (IRBuilderBase &Builder, Value *Vec, unsigned SubVecVF, unsigned Index)
	Generates subvector extract.
SmallBitVector	buildUseMask (int VF, ArrayRef< int > Mask, UseMask MaskArg)
	Prepares a use bitset for the given mask either for the first argument or for the second.
template<bool IsPoisonOnly>
SmallBitVector	isUndefVector (const Value *V, const SmallBitVector &UseMask={})
	Checks if the given value is actually an undefined constant vector.
template SmallBitVector	isUndefVector< false > (const Value *, const SmallBitVector &)
template SmallBitVector	isUndefVector< true > (const Value *, const SmallBitVector &)
bool	doesInTreeUserNeedToExtract (Value Scalar, Instruction UserInst, TargetLibraryInfo TLI, const TargetTransformInfo TTI)
MemoryLocation	getLocation (Instruction *I)
bool	isSimple (Instruction *I)
bool	isSelectedBaseLoad (Type ScalarTy, ArrayRef< Value > PointerOps, const DataLayout &DL, Value &TrueBase, Value &FalseBase, SmallVectorImpl< Value * > &Conditions)
	Checks if the loads with scalar type `ScalarTy` and pointer operands `PointerOps` are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?
Align	computeBlendedLoadBaseAlignment (ArrayRef< Value * > VL, const DataLayout &DL)
	Returns the alignment of the shared base pointer of a blended load for the loads `VL`.
Type *	getCommonGEPIndexType (ArrayRef< Value * > VL, Instruction VL0, function_ref< bool(Value )> IsGEPLane, const DataLayout &DL)
	Returns the common type for the indices of the single-index GEP lanes of a GEP node with the main op `VL0`, or nullptr if no such type exists.
bool	isCopyableGEPAddressVector (ArrayRef< Value * > PointerOps)
	Checks if the pointers `PointerOps` of the gathered loads, which are not compatible in the usual sense (some of them are constant-offset pointers, some have runtime indices), still form a cheap address vector as a copyable GEP node: a splat of the common base and a vector of indices.
void	addMask (SmallVectorImpl< int > &Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs=false)
	Shuffles `Mask` in accordance with the given `SubMask`.
void	fixupOrderingIndices (MutableArrayRef< unsigned > Order)
	Order may have elements assigned special value (size) which is out of bounds.
SmallBitVector	getAltInstrMask (ArrayRef< Value * > VL, Type *ScalarTy, unsigned Opcode0, unsigned Opcode1)
SmallVector< Constant * >	replicateMask (ArrayRef< Constant * > Val, unsigned VF)
	Replicates the given `Val` `VF` times.
Intrinsic::ID	getMaskedDivRemIntrinsic (unsigned Opcode)
static bool	isNonProfitableIndex (const Instruction *I)
	Returns true if `I` is a part of a single-use chain, computing an address, which does not pay off the vectorization: all the lanes are extracted for the scalar addresses, the extracts delay the memory accesses.
bool	isOnceUsedSeed (const Instruction *I)
	Returns true if `I` forms a vectorizable bundle on its own and its single user does not tear the vector apart.
Instruction *	lookThroughCastRoundTrip (Value *V, bool MustBeElidable)
	If `V` is a single-use fpext of a single-use fptrunc forming a round-trip back to the type of `V`, returns the fptrunc; the round-trip source is its operand, always an instruction of the same type as `V`.
static void	collectNarrowedLeavesImpl (Value V, unsigned RdxOpcode, unsigned WideBW, NarrowedChainState S, unsigned Depth, unsigned MaxDepth, SmallVectorImpl< NarrowedLeafInfo > &Leaves, SmallVectorImpl< Instruction > &ChainInsts)
void	collectNarrowedLeaves (Value V, unsigned RdxOpcode, unsigned WideBW, unsigned MaxDepth, SmallVectorImpl< NarrowedLeafInfo > &Leaves, SmallVectorImpl< Instruction > &ChainInsts)
	Recursively collects the narrow leaves of the widened reduction value `V`.
TargetTransformInfo::TargetCostKind	getSLPCostKind (const Function *F)
static std::optional< std::tuple< Value *, unsigned, unsigned > >	matchExtractedField (Value *V)
	Checks if `V` is a zero-extended sub-field of a wider integer scalar.
std::optional< std::tuple< Value *, unsigned, SmallVector< int > > >	matchGatheredExtractedFields (ArrayRef< Value * > VL, const DataLayout &DL)
	Checks if the values in `VL` are zero-extended sub-fields of the same wider integer scalar.
APInt	getScalarMaxValue (const Value *V, unsigned Depth=0)
	Returns a saturating unsigned upper bound of the scalar V.
std::optional< BitPackInfo >	computeBitPackInfo (unsigned BitWidth, ArrayRef< APInt > PossibleBits, ArrayRef< uint64_t > ShlAmts, ArrayRef< APInt > Masks)
	Computes the bitfield packing layout from the per-lane possibly set bits of the source values, the per-lane left-shift amounts and the per-lane masks (all-ones for unmasked lanes).
SmallVector< int >	getBitPackMask (const BitPackInfo &Info, unsigned NumBytes, unsigned NumElts, unsigned BytesPerLane)
	Returns the byte shuffle mask packing the per-lane fields of the shifted lanes (BytesPerLane bytes each) into the packed scalar of NumBytes bytes.
Value *	buildBitPack (IRBuilderBase &Builder, Value *X, const BitPackInfo &Info, unsigned ShiftWidth, unsigned &NumInsts)
	Builds the bitfield packing of X per the layout and the shift width.
void	redirectDbgValues (Instruction &From, Value &To)
	The debug values of the erased `From` are kept on its replacement `To`.

Variables
constexpr unsigned	MaxBitPackAnalysisDepth = MaxAnalysisRecursionDepth + 2
	Deeper than the standard analysis recursion depth to keep the numeric bound precise through arithmetic carry chains.
constexpr int	UsesLimit = 64
	Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Detailed Description

A private "module" namespace for types and utilities used by this pass.

These are implementation details and should not be used by clients.

Enumeration Type Documentation

◆ BoolBitmask

enum class llvm::slpvectorizer::BoolBitmask

strong

The result of matching a boolean bitmask reduction over narrowed leaves.

Enumerator
None
NoMask
NeedMask

Definition at line 38 of file SLPReductionUtils.h.

◆ UseMask

enum class llvm::slpvectorizer::UseMask

strong

Specifies the way the mask should be analyzed for undefs/poisonous elements in the shuffle mask.

Enumerator

FirstArg

The mask is expected to be for permutation of 1-2 vectors, check for the mask elements for the first argument (mask indices are in range [0:VF)).

SecondArg

The mask is expected to be for permutation of 2 vectors, check for the mask elements for the second argument (mask indices are in range [VF:2*VF))

UndefsAsMask

Consider undef mask elements (-1) as placeholders for future shuffle elements and mark them as ones as being used in future.

Non-undef elements are considered as unused since they're already marked as used in the mask.

Definition at line 300 of file SLPUtils.h.

Function Documentation

◆ addMask()

void llvm::slpvectorizer::addMask	(	SmallVectorImpl< int > &	Mask,
		ArrayRef< int >	SubMask,
		bool	ExtendingManyInputs = false )

Shuffles Mask in accordance with the given SubMask.

Parameters

ExtendingManyInputs Supports reshuffling of the mask with not only one but two input vectors.

Definition at line 983 of file SLPUtils.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ allConstant()

bool llvm::slpvectorizer::allConstant ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are constants (but not globals/constant expressions).

Definition at line 152 of file SLPUtils.cpp.

References llvm::all_of(), and isConstant().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getSpillCost(), slpvectorizer::BoUpSLP::hasNonConstantGathers(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameBlock()

bool llvm::slpvectorizer::allSameBlock ( ArrayRef< Value * > VL )

Returns: True if all of the instructions in VL are in the same block.

Definition at line 130 of file SLPUtils.cpp.

References llvm::all_of(), llvm::cast(), llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), II, llvm::IsaPred, isVectorLikeInstWithConstOps(), llvm::iterator_range(), and llvm::make_filter_range().

Referenced by checkEVsForVecCalls(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ allSameOpcode()

bool llvm::slpvectorizer::allSameOpcode ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL use the same opcode. For comparison instructions, also checks if predicates match. PoisonValues are considered matching. Interchangeable instructions are not considered.

Definition at line 274 of file SLPUtils.cpp.

References llvm::all_of(), llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::Instruction::getOpcode(), llvm::isa(), llvm::IsaPred, and llvm::make_range().

Referenced by getSameOpcode().

◆ allSameType()

bool llvm::slpvectorizer::allSameType ( ArrayRef< Value * > VL )

Returns: True iff every value in VL has the same Type as the first.

Definition at line 432 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::consume_front(), and llvm::ArrayRef< T >::empty().

Referenced by slpvectorizer::BoUpSLP::buildTree(), slpvectorizer::BoUpSLP::buildTree(), and slpvectorizer::BoUpSLP::getReorderingData().

◆ areAllOperandsNonInsts()

bool llvm::slpvectorizer::areAllOperandsNonInsts ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all operands are either not instructions or phi nodes or instructions from different blocks.

Definition at line 472 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, llvm::make_isa_range(), and llvm::mayHaveNonDefUseDependency().

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ areCompatibleCmpOps()

bool llvm::slpvectorizer::areCompatibleCmpOps	(	Value *	BaseOp0,
		Value *	BaseOp1,
		Value *	Op0,
		Value *	Op1,
		const TargetLibraryInfo &	TLI )

static

Checks if the provided operands of 2 cmp instructions are compatible, i.e.

compatible instructions or constants, or just some other regular values.

Definition at line 711 of file SLPCompatibilityAnalysis.cpp.

References getSameOpcode(), llvm::isa(), and isConstant().

Referenced by isCmpSameOrSwapped().

◆ arePointersCompatible()

bool llvm::slpvectorizer::arePointersCompatible	(	Value *	Ptr1,
		Value *	Ptr2,
		const TargetLibraryInfo &	TLI,
		unsigned	MaxDepth,
		bool	CompareOpcodes )

MaxDepth is the recursion limit for getUnderlyingObject.

Definition at line 69 of file SLPMemoryUtils.cpp.

References llvm::dyn_cast(), getSameOpcode(), llvm::getUnderlyingObject(), and isConstant().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ buildBitPack()

Value * llvm::slpvectorizer::buildBitPack	(	IRBuilderBase &	Builder,
		Value *	X,
		const BitPackInfo &	Info,
		unsigned	ShiftWidth,
		unsigned &	NumInsts )

Builds the bitfield packing of X per the layout and the shift width.

NumInsts returns the number of emitted instructions.

Definition at line 1476 of file SLPUtils.cpp.

References A(), assert(), llvm::BitWidth, llvm::cast(), llvm::dyn_cast(), llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::IntegerType::get(), llvm::PoisonValue::get(), getBitPackMask(), llvm::Constant::getNullValue(), getScalarSizeInBits(), llvm::is_contained(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::ShuffleVectorInst::isReverseMask(), llvm::slpvectorizer::BitPackInfo::NoLane, llvm::SmallVectorTemplateBase< T, bool >::push_back(), X, and Y.

◆ buildCompressMask()

bool llvm::slpvectorizer::buildCompressMask	(	ArrayRef< Value * >	PointerOps,
		ArrayRef< unsigned >	Order,
		Type *	ScalarTy,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		SmallVectorImpl< int > &	CompressMask )

static

Builds compress-like mask for shuffles for the given PointerOps, ordered with Order.

Returns: true if the mask represents strided access, false - otherwise.

Definition at line 201 of file SLPMemoryUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), llvm::getPointersDiff(), I, llvm::PoisonMaskElem, llvm::seq(), and llvm::ArrayRef< T >::size().

Referenced by isMaskedLoadCompress().

◆ buildUseMask()

SmallBitVector llvm::slpvectorizer::buildUseMask	(	int	VF,
		ArrayRef< int >	Mask,
		UseMask	MaskArg )

Prepares a use bitset for the given mask either for the first argument or for the second.

Definition at line 728 of file SLPUtils.cpp.

References llvm::enumerate(), FirstArg, llvm::PoisonMaskElem, SecondArg, and UndefsAsMask.

Referenced by llvm::slpvectorizer::BaseShuffleAnalysis::createShuffle(), llvm::slpvectorizer::BaseShuffleAnalysis::peekThroughShuffles(), and performExtractsShuffleAction().

◆ calculateRtStride()

const SCEV * llvm::slpvectorizer::calculateRtStride	(	ArrayRef< Value * >	PointerOps,
		Type *	ElemTy,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		SmallVectorImpl< unsigned > &	SortedIndices )

Checks if the provided list of pointers Pointers represents the strided pointers for type ElemTy.

If they are not, nullptr is returned. Otherwise, SCEV* of the stride value is returned. If PointerOps can be rearranged into the following sequence:

%x + c_0 * stride,
%x + c_1 * stride,
%x + c_2 * stride
...

where each c_i is constant. The SCEV of the stride will be returned.

Definition at line 97 of file SLPMemoryUtils.cpp.

References llvm::SmallVectorImpl< T >::clear(), DL, llvm::dyn_cast(), llvm::enumerate(), llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getMinusSCEV(), llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getType(), llvm::ScalarEvolution::getUDivExactExpr(), llvm::isa(), llvm::SCEV::isNonConstantNegative(), llvm::SCEV::isZero(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), Size, llvm::ArrayRef< T >::size(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by slpvectorizer::BoUpSLP::analyzeRtStrideCandidate().

◆ calculateShufflevectorMask()

SmallVector< int > llvm::slpvectorizer::calculateShufflevectorMask ( ArrayRef< Value * > VL )

Returns: a shufflevector mask which is used to vectorize shufflevectors e.g., %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> the result is <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>

Definition at line 561 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::ArrayRef< T >::front(), getShufflevectorNumGroups(), and llvm::PoisonMaskElem.

◆ canWidenLoadsOverUndefLanes()

bool llvm::slpvectorizer::canWidenLoadsOverUndefLanes	(	ArrayRef< Value * >	VL,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		AssumptionCache &	AC,
		const DominatorTree &	DT,
		const TargetLibraryInfo &	TLI )

Checks if the simple loads of VL, mixed with the undef and poison lanes, can be replaced by a single load of the whole VL width.

The loads must be consecutive and match their lanes, and the whole accessed range must be safe to load unconditionally.

Definition at line 392 of file SLPMemoryUtils.cpp.

References llvm::sampleprof::Base, llvm::ArrayRef< T >::begin(), llvm::BitWidth, llvm::cast(), llvm::Instruction::comesBefore(), llvm::count_if(), DL, llvm::dyn_cast(), llvm::End, llvm::enumerate(), llvm::find_if(), llvm::getPointersDiff(), llvm::Value::getType(), llvm::isa(), llvm::IsaPred, llvm::isDereferenceableAndAlignedPointer(), llvm::mustSuppressSpeculation(), llvm::Offset, llvm::ArrayRef< T >::size(), and llvm::Value::stripAndAccumulateConstantOffsets().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ clusterSortPtrAccesses()

bool llvm::slpvectorizer::clusterSortPtrAccesses	(	ArrayRef< Value * >	VL,
		ArrayRef< BasicBlock * >	BBs,
		Type *	ElemTy,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		unsigned	MaxDepth,
		SmallVectorImpl< unsigned > &	SortedIndices )

Clusters VL pointers by (basic block, underlying object) pair and sorts each cluster by offset.

Returns false and leaves SortedIndices empty if the accesses are not worth reordering.

Definition at line 495 of file SLPMemoryUtils.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::sampleprof::Base, llvm::SmallVectorImpl< T >::clear(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::Depth, DL, llvm::ArrayRef< T >::drop_front(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::enumerate(), llvm::ArrayRef< T >::front(), llvm::getUnderlyingObject(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::SmallPtrSetImpl< PtrType >::insert(), P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::stable_sort(), T, and llvm::V1.

Referenced by slpvectorizer::BoUpSLP::findPartiallyOrderedLoads().

◆ collectNarrowedLeaves()

void llvm::slpvectorizer::collectNarrowedLeaves	(	Value *	V,
		unsigned	RdxOpcode,
		unsigned	WideBW,
		unsigned	MaxDepth,
		SmallVectorImpl< NarrowedLeafInfo > &	Leaves,
		SmallVectorImpl< Instruction * > &	ChainInsts )

Recursively collects the narrow leaves of the widened reduction value V.

zext is looked through directly, same-kind binops per operand, shl of a zext - only if no bits are shifted out in the current type, shls in narrower types fold into the shift and ands with a constant into the mask applied in the narrow type. Also collects the looked-through instructions into ChainInsts.

Definition at line 1236 of file SLPUtils.cpp.

References collectNarrowedLeavesImpl().

Referenced by matchPackedFields().

◆ collectNarrowedLeavesImpl()

void llvm::slpvectorizer::collectNarrowedLeavesImpl	(	Value *	V,
		unsigned	RdxOpcode,
		unsigned	WideBW,
		NarrowedChainState	S,
		unsigned	Depth,
		unsigned	MaxDepth,
		SmallVectorImpl< NarrowedLeafInfo > &	Leaves,
		SmallVectorImpl< Instruction * > &	ChainInsts )

static

Definition at line 1154 of file SLPUtils.cpp.

References collectNarrowedLeavesImpl(), llvm::Depth, llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::APInt::getAllOnes(), llvm::APInt::getLowBitsSet(), llvm::APInt::getZExtValue(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::APInt::ult(), and X.

Referenced by collectNarrowedLeaves(), and collectNarrowedLeavesImpl().

◆ combineOrders()

void llvm::slpvectorizer::combineOrders	(	MutableArrayRef< unsigned >	Order,
		ArrayRef< unsigned >	SecondaryOrder )

Fills unset elements of Order (marked with the sentinel value equal to the order size) with the corresponding elements of SecondaryOrder, skipping already used indices, or with the identity order if SecondaryOrder is empty.

Definition at line 410 of file SLPUtils.cpp.

References assert(), llvm::ArrayRef< T >::empty(), llvm::seq(), llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::SmallBitVector::test().

Referenced by slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ computeBitPackInfo()

std::optional< BitPackInfo > llvm::slpvectorizer::computeBitPackInfo	(	unsigned	BitWidth,
		ArrayRef< APInt >	PossibleBits,
		ArrayRef< uint64_t >	ShlAmts,
		ArrayRef< APInt >	Masks )

Computes the bitfield packing layout from the per-lane possibly set bits of the source values, the per-lane left-shift amounts and the per-lane masks (all-ones for unmasked lanes).

Definition at line 1430 of file SLPUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::BitWidth, llvm::APInt::isShiftedMask(), llvm::APInt::isZero(), llvm::Lo, llvm::slpvectorizer::BitPackInfo::LShrAmts, llvm::slpvectorizer::BitPackInfo::NoLane, llvm::seq(), and llvm::ArrayRef< T >::size().

◆ computeBlendedLoadBaseAlignment()

Align llvm::slpvectorizer::computeBlendedLoadBaseAlignment	(	ArrayRef< Value * >	VL,
		const DataLayout &	DL )

Returns the alignment of the shared base pointer of a blended load for the loads VL.

Definition at line 881 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::cast(), llvm::commonAlignment(), DL, llvm::enumerate(), llvm::ArrayRef< T >::front(), getAlign(), and llvm::IsaPred.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ computeCommonAlignment()

template<typename T>

Align llvm::slpvectorizer::computeCommonAlignment ( ArrayRef< Value * > VL )

Calculates minimal alignment as a common alignment.

Definition at line 87 of file SLPMemoryUtils.cpp.

References llvm::cast(), llvm::ArrayRef< T >::consume_front(), and getAlign().

Referenced by tryToFindDuplicates().

◆ computeCommonAlignment< LoadInst >()

template Align llvm::slpvectorizer::computeCommonAlignment< LoadInst > ( ArrayRef< Value * > )

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ computeCommonAlignment< StoreInst >()

template Align llvm::slpvectorizer::computeCommonAlignment< StoreInst > ( ArrayRef< Value * > )

◆ convertTo()

std::pair< Instruction , SmallVector< Value > > llvm::slpvectorizer::convertTo	(	Instruction *	I,
		const InstructionsState &	S )

Returns: the main or alternate operation from S matching I, together with the operands of I adjusted to the selected operation.

Definition at line 972 of file SLPCompatibilityAnalysis.cpp.

References assert(), C(), llvm::cast(), Converter, llvm::dyn_cast(), llvm::slpvectorizer::CmpSamePredicateHelper::getAdjustedConstant(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), I, and llvm::slpvectorizer::InstructionsState::isAltShuffle().

◆ createExtractVector()

Value * llvm::slpvectorizer::createExtractVector	(	IRBuilderBase &	Builder,
		Value *	Vec,
		unsigned	SubVecVF,
		unsigned	Index )

Generates subvector extract.

Definition at line 721 of file SLPUtils.cpp.

References llvm::PoisonMaskElem.

Referenced by slpvectorizer::BoUpSLP::vectorizeTree().

◆ createInsertVector()

Value * llvm::slpvectorizer::createInsertVector	(	IRBuilderBase &	Builder,
		Value *	Vec,
		Value *	V,
		unsigned	Index,
		function_ref< Value (Value , Value *, ArrayRef< int >)>	Generator = {} )

Creates subvector insert.

Generates shuffle using Generator or using default shuffle.

Definition at line 692 of file SLPUtils.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), getNumElements(), llvm::Value::getType(), llvm::isa(), and llvm::PoisonMaskElem.

◆ createWidenedStridedCast()

Value * llvm::slpvectorizer::createWidenedStridedCast	(	IRBuilderBase &	Builder,
		Value *	V,
		Type *	DstTy,
		const DataLayout &	DL )

Casts V between the widened strided access type and the entry vector type DstTy.

Definition at line 46 of file SLPMemoryUtils.cpp.

References llvm::cast(), and DL.

◆ doesInTreeUserNeedToExtract()

bool llvm::slpvectorizer::doesInTreeUserNeedToExtract	(	Value *	Scalar,
		Instruction *	UserInst,
		TargetLibraryInfo *	TLI,
		const TargetTransformInfo *	TTI )

Returns: True if in-tree use also needs extract. This refers to possible scalar operand in vectorized instruction.

Definition at line 795 of file SLPUtils.cpp.

References llvm::any_of(), llvm::CallBase::args(), llvm::cast(), llvm::enumerate(), llvm::Instruction::getOpcode(), llvm::LoadInst::getPointerOperand(), and llvm::getVectorIntrinsicIDForCall().

Referenced by slpvectorizer::BoUpSLP::buildExternalUses().

◆ doesNotNeedToBeScheduled()

bool llvm::slpvectorizer::doesNotNeedToBeScheduled ( Value * V )

Checks if the specified value does not require scheduling.

It does not require scheduling if all operands and all users do not need to be scheduled in the current basic block.

Definition at line 497 of file SLPUtils.cpp.

References areAllOperandsNonInsts(), and isUsedOutsideBlock().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and llvm::slpvectorizer::InstructionsState::isNonSchedulable().

◆ doesNotNeedToSchedule()

bool llvm::slpvectorizer::doesNotNeedToSchedule ( ArrayRef< Value * > VL )

Checks if the specified array of instructions does not require scheduling.

It is so if all either instructions have operands that do not require scheduling or their users do not require scheduling since they are phis or in other basic blocks.

Definition at line 501 of file SLPUtils.cpp.

References llvm::all_of(), areAllOperandsNonInsts(), llvm::ArrayRef< T >::empty(), and isUsedOutsideBlock().

Referenced by tryToFindDuplicates().

◆ findInstructionWithOpcode()

Instruction * llvm::slpvectorizer::findInstructionWithOpcode	(	ArrayRef< Value * >	VL,
		unsigned	Opcode )

static

Find an instruction with a specific opcode in VL.

Parameters

VL	Array of values to search through. Must contain only Instructions and PoisonValues.
Opcode	The instruction opcode to search for

Returns

The first instruction found with matching opcode
nullptr if no matching instruction is found

Definition at line 697 of file SLPCompatibilityAnalysis.cpp.

References assert(), llvm::cast(), llvm::isa(), and llvm::make_filter_range().

Referenced by getSameOpcode().

◆ fixupOrderingIndices()

void llvm::slpvectorizer::fixupOrderingIndices ( MutableArrayRef< unsigned > Order )

Order may have elements assigned special value (size) which is out of bounds.

Such indices only appear on places which correspond to undef values (see canReuseExtract for details) and used in order to avoid undef values have effect on operands ordering. The first loop below simply finds all unused indices and then the next loop nest assigns these indices for undef values positions. As an example below Order has two undef positions and they have assigned values 3 and 7 respectively: before: 6 9 5 4 9 2 1 0 after: 6 3 5 4 7 2 1 0

Definition at line 1008 of file SLPUtils.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::SmallBitVector::find_first(), llvm::SmallBitVector::find_next(), I, llvm::SmallBitVector::none(), llvm::SmallBitVector::reset(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getAltInstrMask()

SmallBitVector llvm::slpvectorizer::getAltInstrMask	(	ArrayRef< Value * >	VL,
		Type *	ScalarTy,
		unsigned	Opcode0,
		unsigned	Opcode1 )

Returns: a bitset for selecting opcodes. false for Opcode0 and true for Opcode1.

Definition at line 1032 of file SLPUtils.cpp.

References llvm::cast(), getNumElements(), llvm::isa(), llvm::seq(), llvm::SmallBitVector::set(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getBitPackCost()

InstructionCost llvm::slpvectorizer::getBitPackCost	(	const TargetTransformInfo &	TTI,
		FixedVectorType *	SrcTy,
		Type *	ResultTy,
		const BitPackInfo &	Info,
		unsigned	ZExtSrcWidth,
		TargetTransformInfo::CastContextHint	CCH,
		TargetTransformInfo::TargetCostKind	CostKind,
		const TargetLibraryInfo *	TLI,
		const Instruction *	CtxI,
		unsigned &	ShiftWidth )

Returns the cost of the bitfield packing of SrcTy into ResultTy, picking the cheapest shift width.

The packing is a trunc, an lshr, a byte shuffle and a bitcast. ZExtSrcWidth is the source width of the lanes if they are a plain zext (0 otherwise), so compacting them back to it is free. CCH is the context of the pack's source operand.

Definition at line 411 of file SLPCostAnalysis.cpp.

References A(), llvm::all_equal(), llvm::all_of(), assert(), llvm::BitWidth, C(), CostKind, llvm::FixedVectorType::get(), llvm::IntegerType::get(), getBitPackMask(), llvm::Type::getInt8Ty(), llvm::InstructionCost::getInvalid(), llvm::is_contained(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::ShuffleVectorInst::isReverseMask(), llvm::InstructionCost::isValid(), llvm::max_element(), llvm::slpvectorizer::BitPackInfo::NoLane, llvm::TargetTransformInfo::OK_NonUniformConstantValue, llvm::TargetTransformInfo::OK_UniformConstantValue, llvm::TargetTransformInfo::OP_None, llvm::TargetTransformInfo::OP_PowerOf2, llvm::TargetTransformInfo::SK_PermuteSingleSrc, and llvm::TargetTransformInfo::SK_PermuteTwoSrc.

◆ getBitPackMask()

SmallVector< int > llvm::slpvectorizer::getBitPackMask	(	const BitPackInfo &	Info,
		unsigned	NumBytes,
		unsigned	NumElts,
		unsigned	BytesPerLane )

Returns the byte shuffle mask packing the per-lane fields of the shifted lanes (BytesPerLane bytes each) into the packed scalar of NumBytes bytes.

Definition at line 1463 of file SLPUtils.cpp.

References llvm::slpvectorizer::BitPackInfo::NoLane, and llvm::seq().

Referenced by buildBitPack(), and getBitPackCost().

◆ getBlendedLoadCost()

InstructionCost llvm::slpvectorizer::getBlendedLoadCost	(	const TargetTransformInfo &	TTI,
		Type *	VecTy,
		Align	Alignment,
		unsigned	AddressSpace,
		const TargetTransformInfo::TargetCostKind	CostKind )

Returns the cost of a BlendedLoadVectorize node loading VecTy: two masked loads (one per candidate base), a xor to negate the false-lane mask and a select.

The blend mask is a separate operand node, so its cost is counted there, not here.

Definition at line 158 of file SLPCostAnalysis.cpp.

References llvm::CmpInst::BAD_ICMP_PREDICATE, CostKind, and llvm::CmpInst::makeCmpResultType().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ getBoolBitmaskCost()

InstructionCost llvm::slpvectorizer::getBoolBitmaskCost	(	const TargetTransformInfo &	TTI,
		bool	NeedMask,
		Type *	NarrowScalarTy,
		Type *	WideTy,
		unsigned	VF,
		ArrayRef< int >	PermMask,
		const Value *	Root,
		TargetTransformInfo::TargetCostKind	CostKind )

Returns the cost of the boolean bitmask reduction of a vector of boolean leaves of type NarrowScalarTy, emitted as [and] + [lane permutation PermMask] + zero test + bitcast [+ zext] to WideTy.

Root is the reduction root, used as the context of the emitted instructions.

Definition at line 479 of file SLPCostAnalysis.cpp.

References llvm::cast(), CostKind, llvm::IntegerType::get(), getBoolReduxResultCCH(), llvm::Type::getContext(), getShuffleCost(), getWidenedType(), llvm::CmpInst::ICMP_NE, llvm::ShuffleVectorInst::isIdentityMask(), llvm::Type::isIntegerTy(), llvm::CmpInst::makeCmpResultType(), NeedMask, llvm::TargetTransformInfo::None, llvm::TargetTransformInfo::OK_AnyValue, llvm::TargetTransformInfo::OK_NonUniformConstantValue, llvm::TargetTransformInfo::OK_UniformConstantValue, llvm::TargetTransformInfo::OP_None, and llvm::TargetTransformInfo::SK_PermuteSingleSrc.

◆ getBoolLogicRdxBitcastCost()

InstructionCost llvm::slpvectorizer::getBoolLogicRdxBitcastCost	(	RecurKind	Kind,
		const TargetTransformInfo &	TTI,
		FixedVectorType *	VectorTy,
		TTI::CastContextHint	Ctx,
		TTI::TargetCostKind	CostKind )

static

Definition at line 349 of file SLPCostAnalysis.cpp.

References llvm::And, assert(), CostKind, llvm::IntegerType::get(), llvm::Type::getContext(), llvm::VectorType::getElementType(), getNumElements(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::Type::isIntegerTy(), llvm::CmpInst::makeCmpResultType(), llvm::TargetTransformInfo::OK_UniformConstantValue, llvm::TargetTransformInfo::OP_None, and llvm::Or.

Referenced by getI1ReductionCost().

◆ getBoolReduxBitcastCmpCost()

InstructionCost llvm::slpvectorizer::getBoolReduxBitcastCmpCost	(	const TargetTransformInfo &	TTI,
		RecurKind	RdxKind,
		FixedVectorType *	VecTy,
		const Value *	Root,
		ArrayRef< Instruction * >	ChainInsts,
		const TTI::TargetCostKind	CostKind )

Returns the cost of the booleanized logical and/or reduction of a vector of type VecTy with the i1 root Root, emitted as trunc+bitcast+cmp, estimated in the context of the replaced cast chain ChainInsts.

Definition at line 315 of file SLPCostAnalysis.cpp.

References llvm::And, CostKind, llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::FixedVectorType::get(), llvm::IntegerType::get(), llvm::Constant::getAllOnesValue(), llvm::Type::getContext(), llvm::Type::getInt1Ty(), llvm::Constant::getNullValue(), llvm::FixedVectorType::getNumElements(), I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::isa(), and llvm::CmpInst::makeCmpResultType().

◆ getBoolReduxResultCCH()

TTI::CastContextHint llvm::slpvectorizer::getBoolReduxResultCCH ( const Value * Root )

static

Returns the cast context hint for the trunc of the booleanized reduction result, which inherits the uses of the reduction root Root.

Definition at line 290 of file SLPCostAnalysis.cpp.

References llvm::TargetTransformInfo::GatherScatter, llvm::Value::hasOneUse(), llvm::isa(), llvm::PatternMatch::m_Intrinsic(), llvm::TargetTransformInfo::Masked, llvm::PatternMatch::match(), llvm::TargetTransformInfo::None, llvm::TargetTransformInfo::Normal, and llvm::Value::user_begin().

Referenced by getBoolBitmaskCost(), and getBoolReduxWideRdxCost().

◆ getBoolReduxWideRdxCost()

InstructionCost llvm::slpvectorizer::getBoolReduxWideRdxCost	(	const TargetTransformInfo &	TTI,
		RecurKind	RdxKind,
		FixedVectorType *	VecTy,
		const Value *	Root,
		FastMathFlags	FMF,
		const TTI::TargetCostKind	CostKind )

Returns the cost of the booleanized logical and/or reduction of a vector of type VecTy with the i1 root Root, emitted as the wide reduction plus the result trunc.

Definition at line 303 of file SLPCostAnalysis.cpp.

References CostKind, getBoolReduxResultCCH(), llvm::Type::getContext(), llvm::Type::getInt1Ty(), llvm::RecurrenceDescriptor::getOpcode(), and llvm::Type::getScalarType().

Referenced by tryEmitBoolReduxBitcastCmp().

◆ getBoolReduxWideTy()

Type * llvm::slpvectorizer::getBoolReduxWideTy	(	RecurKind	RdxKind,
		Type *	RootTy,
		Type *	LeafTy )

Returns: the wide leaf type if the logical and/or reduction RdxKind with the i1 root type RootTy and the leaf type LeafTy is a booleanized reduction (performed in the wide leaf type, bit 0 of the result is the final value), nullptr otherwise.

Definition at line 68 of file SLPReductionUtils.cpp.

References llvm::And, llvm::Type::isIntegerTy(), and llvm::Or.

◆ getCmpBoundaryFamily()

std::optional< std::pair< bool, APInt > > llvm::slpvectorizer::getCmpBoundaryFamily	(	CmpInst::Predicate	Pred,
		const APInt &	C )

static

If the comparison (Pred, X, C) is a single-element or single-complement range check, returns its boundary family: false + K for the singleton {K} (eq forms), true + K for the complement of {K} (ne forms).

Definition at line 333 of file SLPCompatibilityAnalysis.cpp.

References C(), llvm::APInt::getMaxValue(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, and llvm::CmpInst::isSigned().

Referenced by llvm::slpvectorizer::CmpSamePredicateHelper::getAdjustedConstant().

◆ getCommonGEPIndexType()

Type * llvm::slpvectorizer::getCommonGEPIndexType	(	ArrayRef< Value * >	VL,
		Instruction *	VL0,
		function_ref< bool(Value *)>	IsGEPLane,
		const DataLayout &	DL )

Returns the common type for the indices of the single-index GEP lanes of a GEP node with the main op VL0, or nullptr if no such type exists.

IsGEPLane tells which lanes of VL are matching GEPs, whose index is used as is; all other lanes (copyable, poison, non-GEP pointers) are modeled as gep V, 0 and just take a zero index of the common type. The common type is the index type of VL0 if all matching lanes share it. Otherwise the constant indices are cast: to the pointer index type if there are no non-constant indices (or the index type of VL0 already is the pointer index type), or to the index type of VL0 if all the non-constant indices have that type and all the constants are representable in it (GEP indices are sign-extended to the pointer index width, so the value must fit as a signed number). A non-constant index of a different type cannot be cast without a new instruction, so no common type exists.

Definition at line 894 of file SLPUtils.cpp.

References llvm::cast(), DL, llvm::dyn_cast(), llvm::Type::getIntegerBitWidth(), llvm::User::getOperand(), llvm::Type::getScalarType(), llvm::Value::getType(), and llvm::make_filter_range().

◆ getElementIndex()

std::optional< unsigned > llvm::slpvectorizer::getElementIndex	(	const Value *	Inst,
		unsigned	Offset = 0 )

Returns: inserting or extracting index of InsertElement, ExtractElement or InsertValue instruction, using Offset as base offset for index.; std::nullopt if the index is not an immediate.

Definition at line 246 of file SLPUtils.cpp.

References llvm::dyn_cast(), getInsertExtractIndex(), I, IV, and llvm::Offset.

Referenced by areTwoInsertFromSameBuildVector(), slpvectorizer::BoUpSLP::getReorderingData(), and isUndefVector().

◆ getExtractIndex()

std::optional< unsigned > llvm::slpvectorizer::getExtractIndex ( const Instruction * E )

Returns: Optional element Idx for Extract{Value,Element} instructions.

Definition at line 292 of file SLPUtils.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), and getNumElements().

Referenced by slpvectorizer::BoUpSLP::getReorderingData().

◆ getExtractWithExtendCost()

InstructionCost llvm::slpvectorizer::getExtractWithExtendCost	(	const TargetTransformInfo &	TTI,
		bool	ReVec,
		unsigned	Opcode,
		Type *	Dst,
		VectorType *	VecTy,
		unsigned	Index,
		const TTI::TargetCostKind	CostKind )

This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst is a FixedVectorType, a vector will be extracted instead of a scalar.

Definition at line 272 of file SLPCostAnalysis.cpp.

References assert(), llvm::cast(), CostKind, getNumElements(), getShuffleCost(), getWidenedType(), llvm::isVectorizedTy(), llvm::TargetTransformInfo::None, llvm::TargetTransformInfo::SK_ExtractSubvector, and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

◆ getFloorFullVectorNumberOfElements()

unsigned llvm::slpvectorizer::getFloorFullVectorNumberOfElements	(	const TargetTransformInfo &	TTI,
		Type *	Ty,
		unsigned	Sz,
		bool	ReVec )

Returns the number of elements of the given type Ty, not greater than Sz, which forms type, which splits by TTI into whole vector types during legalization.

Definition at line 88 of file SLPTypeUtils.cpp.

References llvm::bit_ceil(), llvm::bit_floor(), llvm::divideCeil(), getWidenedType(), llvm::isa(), and isValidElementType().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ getFullVectorNumberOfElements()

unsigned llvm::slpvectorizer::getFullVectorNumberOfElements	(	const TargetTransformInfo &	TTI,
		Type *	Ty,
		unsigned	Sz,
		bool	ReVec )

Returns the number of elements of the given type Ty, not less than Sz, which forms type, which splits by TTI into whole vector types during legalization.

Definition at line 77 of file SLPTypeUtils.cpp.

References llvm::bit_ceil(), llvm::divideCeil(), getWidenedType(), llvm::isa(), and isValidElementType().

Referenced by getMaskedDivRemType(), getNumberOfPartsOrRegs(), isMaskedLoadCompress(), and tryToFindDuplicates().

◆ getGEPCosts()

std::pair< InstructionCost, InstructionCost > llvm::slpvectorizer::getGEPCosts	(	const TargetTransformInfo &	TTI,
		ArrayRef< Value * >	Ptrs,
		Value *	BasePtr,
		unsigned	Opcode,
		const TTI::TargetCostKind	CostKind,
		Type *	ScalarTy,
		VectorType *	VecTy )

Calculate the scalar and the vector costs from vectorizing set of GEPs.

Definition at line 66 of file SLPCostAnalysis.cpp.

References llvm::all_equal(), llvm::all_of(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::ArrayRef< T >::front(), llvm::isa(), llvm::IsaPred, llvm::TargetTransformInfo::PointersChainInfo::IsSameBaseAddress, llvm::map_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::TargetTransformInfo::TCC_Free.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

◆ getI1ReductionCost()

std::pair< InstructionCost, bool > llvm::slpvectorizer::getI1ReductionCost	(	RecurKind	Kind,
		const TargetTransformInfo &	TTI,
		FixedVectorType *	VectorTy,
		Type *	ScalarTy,
		TargetTransformInfo::CastContextHint	Ctx,
		TargetTransformInfo::TargetCostKind	CostKind )

i1 reductions can be emitted as the plain target reduction or in the bitcast-based form (bitcast to a scalar integer type plus a compare for and/or, plus ctpop for add).

Returns the cost of the cheaper form and whether it is the bitcast-based one. Ties keep the historically default form: plain for and/or, bitcast-based for add.

Definition at line 369 of file SLPCostAnalysis.cpp.

References llvm::Add, llvm::And, assert(), CostKind, llvm::IntegerType::get(), llvm::VectorType::get(), getBoolLogicRdxBitcastCost(), llvm::Type::getContext(), llvm::Type::getIntegerBitWidth(), getNumElements(), llvm::RecurrenceDescriptor::getOpcode(), llvm::Type::isIntegerTy(), and llvm::Or.

◆ getInsertExtractIndex()

template<typename T>

std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex	(	const Value *	Inst,
		unsigned	Offset )

Returns: inserting or extracting index of InsertElement / ExtractElement instruction, using Offset as base offset for index. Only instantiated for InsertElementInst and ExtractElementInst (see SLPUtils.cpp).

Definition at line 439 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::Offset.

Referenced by getElementIndex().

◆ getInsertExtractIndex< ExtractElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< ExtractElementInst >	(	const Value *	,
		unsigned	)

◆ getInsertExtractIndex< InsertElementInst >()

template std::optional< unsigned > llvm::slpvectorizer::getInsertExtractIndex< InsertElementInst >	(	const Value *	,
		unsigned	)

◆ getLoadOfUndefTolerantBinOp()

const LoadInst * llvm::slpvectorizer::getLoadOfUndefTolerantBinOp ( const Value * V )

Returns: the load operand of the undef tolerant binary operator V, if any.

Definition at line 55 of file SLPUtils.cpp.

References llvm::cast(), llvm::dyn_cast(), and isUndefTolerantBinOp().

◆ getLocation()

MemoryLocation llvm::slpvectorizer::getLocation ( Instruction * I )

Returns: the AA location that is being access by the instruction.

Definition at line 823 of file SLPUtils.cpp.

References llvm::dyn_cast(), llvm::MemoryLocation::get(), and I.

Referenced by slpvectorizer::BoUpSLP::::HorizontalReduction.

◆ getMaskedDivRemCost()

InstructionCost llvm::slpvectorizer::getMaskedDivRemCost	(	const TargetTransformInfo &	TTI,
		bool	ReVec,
		unsigned	Opcode,
		Type *	ScalarTy,
		unsigned	NumElts,
		const TargetTransformInfo::TargetCostKind	CostKind,
		FixedVectorType **	PaddedTy = nullptr )

For a non-power-of-2 NumElts-wide integer div/rem Opcode, checks if padding to a full register and using the masked div/rem intrinsic is cheaper than the direct vector op.

Returns the cost of the masked alternative, or an invalid cost if it is not applicable or not cheaper.

Definition at line 196 of file SLPCostAnalysis.cpp.

References CostKind, llvm::FixedVectorType::get(), llvm::Type::getContext(), llvm::Type::getInt1Ty(), llvm::InstructionCost::getInvalid(), getMaskedDivRemIntrinsic(), getMaskedDivRemType(), llvm::FixedVectorType::getNumElements(), and getWidenedType().

◆ getMaskedDivRemIntrinsic()

Intrinsic::ID llvm::slpvectorizer::getMaskedDivRemIntrinsic ( unsigned Opcode )

Returns: the masked division/remainder intrinsic corresponding to Opcode. Disabled lanes of these intrinsics are poison rather than UB, unlike the plain opcode.

Definition at line 1055 of file SLPUtils.cpp.

References llvm_unreachable.

Referenced by getMaskedDivRemCost().

◆ getMaskedDivRemType()

FixedVectorType * llvm::slpvectorizer::getMaskedDivRemType	(	const TargetTransformInfo &	TTI,
		unsigned	Opcode,
		Type *	ScalarTy,
		unsigned	NumElts,
		bool	ReVec )

For a non-power-of-2 NumElts-wide integer div/rem Opcode, returns the padded full-register vector type if padding is structurally possible, or nullptr if the vector already fills a register or the opcode is not div/rem.

Does not check profitability.

Definition at line 102 of file SLPTypeUtils.cpp.

References llvm::cast(), getFullVectorNumberOfElements(), getWidenedType(), llvm::has_single_bit(), and llvm::Instruction::isIntDivRem().

Referenced by getMaskedDivRemCost().

◆ getNarrowedLeafOpsCost()

InstructionCost llvm::slpvectorizer::getNarrowedLeafOpsCost	(	const TargetTransformInfo &	TTI,
		const SmallDenseMap< Value *, NarrowedLeafInfo > &	NarrowedLeafShifts,
		VectorType *	NarrowVecTy,
		VectorType *	WideVecTy,
		const Instruction *	CtxI,
		const TTI::TargetCostKind	CostKind )

Returns the cost of the per-lane operations on the narrowed leaves NarrowedLeafShifts: the shl in the wide vector type if any leaf is shifted and the and in the narrow vector type if any leaf is masked.

Definition at line 514 of file SLPCostAnalysis.cpp.

References llvm::any_of(), CostKind, and P.

◆ getNonPhiOperand()

Instruction * llvm::slpvectorizer::getNonPhiOperand	(	Instruction *	I,
		PHINode *	Phi )

Returns: the first operand of I that does not match Phi. If the operand is not an instruction, returns nullptr.

Definition at line 53 of file SLPReductionUtils.cpp.

References llvm::dyn_cast(), I, and matchRdxBop().

◆ getNumberOfPartsOrRegs()

unsigned llvm::slpvectorizer::getNumberOfPartsOrRegs	(	bool	QueryNumParts,
		const TargetTransformInfo &	TTI,
		Type *	VecTy,
		Type *	ScalarTy,
		bool	ReVec,
		unsigned	Limit = std::numeric_limits< unsigned >::max() )

Returns number of parts, the type VecTy will be split at the codegen phase.

If the type is going to be scalarized or does not use whole registers, returns 1.

Definition at line 133 of file SLPTypeUtils.cpp.

References llvm::divideCeil(), getFullVectorNumberOfElements(), getNumElements(), hasFullVectorsOrPowerOf2(), llvm::isa(), and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::getNumberOfParts(), and slpvectorizer::BoUpSLP::getRegUsageForType().

◆ getNumberOfPotentiallyCommutativeOps()

unsigned llvm::slpvectorizer::getNumberOfPotentiallyCommutativeOps ( Instruction * I )

Returns: number of operands of I, considering commutativity. Returns 2 for commutative intrinsics.

Parameters

I	The instruction to check for commutativity

Definition at line 236 of file SLPUtils.cpp.

References I, llvm::isa(), and isCommutative().

Referenced by slpvectorizer::BoUpSLP::::HorizontalReduction.

◆ getNumElements()

unsigned llvm::slpvectorizer::getNumElements ( Type * Ty )

◆ getNumElems()

unsigned llvm::slpvectorizer::getNumElems	(	unsigned	Size,
		unsigned	PartNumElems,
		unsigned	Part )

Returns correct remaining number of elements, considering total amount Size, (power-of-2 number) of elements in a single register PartNumElems and current register (part) Part.

Definition at line 115 of file SLPUtils.cpp.

References Size.

Referenced by slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getPartNumElems()

unsigned llvm::slpvectorizer::getPartNumElems	(	unsigned	Size,
		unsigned	NumParts )

Returns power-of-2 number of elements in a single register (part), given the total number of elements Size and number of registers (parts) NumParts.

Definition at line 111 of file SLPUtils.cpp.

References llvm::bit_ceil(), llvm::divideCeil(), and Size.

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), and slpvectorizer::BoUpSLP::processBuildVector().

◆ getReassocCombineOpcode()

unsigned llvm::slpvectorizer::getReassocCombineOpcode ( unsigned Opcode )

Returns: the opcode of the combines emitted for a reassociated node: subtract chains regroup their positive and negative operand columns with plain adds.

Definition at line 64 of file SLPUtils.cpp.

Referenced by getReassocColumnsQuality(), and scanAssociativeOperands().

◆ getSameOpcode()

InstructionsState llvm::slpvectorizer::getSameOpcode	(	ArrayRef< Value * >	VL,
		const TargetLibraryInfo &	TLI )

Returns: analysis of the Instructions in VL described in InstructionsState, the Opcode that we suppose the whole list could be vectorized even if its structure is diverse.

Definition at line 744 of file SLPCompatibilityAnalysis.cpp.

Referenced by areCompatibleCmpOps(), arePointersCompatible(), checkEVsForVecCalls(), compareCmp(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::isTreeNotExtendable(), and slpvectorizer::BoUpSLP::VLOperands::reorder().

◆ getScalarizationOverhead()

InstructionCost llvm::slpvectorizer::getScalarizationOverhead	(	const TargetTransformInfo &	TTI,
		bool	ReVec,
		Type *	ScalarTy,
		VectorType *	Ty,
		const APInt &	DemandedElts,
		bool	Insert,
		bool	Extract,
		const TTI::TargetCostKind	CostKind,
		bool	ForPoisonSrc,
		ArrayRef< Value * >	VL,
		TTI::VectorInstrContext	VIC )

This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorType, a vector will be inserted or extracted instead of a scalar.

Definition at line 222 of file SLPCostAnalysis.cpp.

References assert(), CostKind, llvm::dyn_cast(), llvm::APInt::getBitWidth(), getNumElements(), getShuffleCost(), I, llvm::isa(), llvm::seq(), llvm::TargetTransformInfo::SK_ExtractSubvector, and llvm::TargetTransformInfo::SK_InsertSubvector.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), isMaskedLoadCompress(), and tryToFindDuplicates().

◆ getScalarMaxValue()

APInt llvm::slpvectorizer::getScalarMaxValue	(	const Value *	V,
		unsigned	Depth = 0 )

Returns a saturating unsigned upper bound of the scalar V.

The numeric bound keeps precision on arithmetic carries, where bit-wise analysis loses it.

Definition at line 1383 of file SLPUtils.cpp.

Referenced by getScalarMaxValue().

◆ getShuffleCost()

InstructionCost llvm::slpvectorizer::getShuffleCost	(	const TargetTransformInfo &	TTI,
		TargetTransformInfo::ShuffleKind	Kind,
		VectorType *	Tp,
		const TargetTransformInfo::TargetCostKind	CostKind,
		ArrayRef< int >	Mask = {},
		int	Index = 0,
		VectorType *	SubTp = nullptr,
		ArrayRef< const Value * >	Args = {},
		TargetTransformInfo::VectorInstrContext	VIC = TargetTransformInfo::VectorInstrContext::None )

Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.

Adds SLP-specific cost estimation for insert subvector pattern.

Definition at line 39 of file SLPCostAnalysis.cpp.

References CostKind, llvm::FixedVectorType::get(), llvm::VectorType::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::Type::getScalarType(), llvm::ShuffleVectorInst::isInsertSubvectorMask(), llvm::TargetTransformInfo::SK_InsertSubvector, and llvm::TargetTransformInfo::SK_PermuteTwoSrc.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canVectorizeLoads(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::createExtractedFieldsVector(), getBoolBitmaskCost(), getExtractWithExtendCost(), slpvectorizer::BoUpSLP::getReorderingData(), getScalarizationOverhead(), slpvectorizer::BoUpSLP::getTreeCost(), getVectorInstrCost(), isMaskedLoadCompress(), tryToFindDuplicates(), and vectorizePackedAggregate().

◆ getShufflevectorNumGroups()

unsigned llvm::slpvectorizer::getShufflevectorNumGroups ( ArrayRef< Value * > VL )

Returns

the number of groups of shufflevector A group has the following features

All of value in a group are shufflevector.
The mask of all shufflevector is isExtractSubvectorMask.
The mask of all shufflevector uses all of the elements of the source. e.g., it is 1 group (%0) %1 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %2 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> it is 2 groups (%3 and %4) %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> it is 0 group %12 = shufflevector <8 x i16> %10, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %13 = shufflevector <8 x i16> %11, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>

Definition at line 521 of file SLPUtils.cpp.

References llvm::SmallBitVector::all(), llvm::all_of(), assert(), llvm::cast(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), I, llvm::IsaPred, llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().

Referenced by calculateShufflevectorMask().

◆ getSLPCostKind()

TargetTransformInfo::TargetCostKind llvm::slpvectorizer::getSLPCostKind ( const Function * F )

Definition at line 1244 of file SLPUtils.cpp.

References assert(), F, llvm::TargetTransformInfo::TCK_CodeSize, and llvm::TargetTransformInfo::TCK_RecipThroughput.

Referenced by slpvectorizer::BoUpSLP::BoUpSLP().

◆ getStrideBytesIfConstant()

ConstantInt * llvm::slpvectorizer::getStrideBytesIfConstant	(	Value *	Stride,
		Type *	ScalarTy,
		const DataLayout &	DL,
		bool	IsReverse )

Returns Stride scaled by the allocation size of ScalarTy, negated if IsReverse is set, or nullptr if Stride is not a constant.

Definition at line 58 of file SLPMemoryUtils.cpp.

References DL, and llvm::dyn_cast_or_null().

◆ getValueType()

Type * llvm::slpvectorizer::getValueType	(	Value *	V,
		bool	ReVec,
		bool	LookThroughCmp = false )

Returns the "element type" of the given value/instruction V.

For stores, returns the stored value type; for insertelement (when ReVec is off), the inserted operand type. For compares, the default is to return the result type (i1); when LookThroughCmp is true, returns the type of the compared operands instead, which is needed for vector width calculations (the width is determined by the operand type, not the i1 result).

Definition at line 40 of file SLPTypeUtils.cpp.

References llvm::dyn_cast(), and IV.

Referenced by blockBodyHasVectorInstructions(), slpvectorizer::BoUpSLP::canBuildSplitNode(), gatherPossiblyVectorizableLoads(), slpvectorizer::BoUpSLP::getInsertBuildVectorSrcTy(), slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::getTreeCost(), slpvectorizer::BoUpSLP::reorderBottomToTop(), slpvectorizer::BoUpSLP::reorderTopToBottom(), tryToFindDuplicates(), tryToVectorizeSequence(), and llvm::SLPVectorizerPass::vectorizeCmpInsts().

◆ getVectorInstrCost()

InstructionCost llvm::slpvectorizer::getVectorInstrCost	(	const TargetTransformInfo &	TTI,
		bool	ReVec,
		Type *	ScalarTy,
		unsigned	Opcode,
		Type *	Val,
		const TTI::TargetCostKind	CostKind,
		unsigned	Index,
		Value *	Scalar,
		ArrayRef< std::tuple< Value , User , int > >	ScalarUserAndIdx,
		TTI::VectorInstrContext	VIC )

This is similar to TargetTransformInfo::getVectorInstrCost, but if ScalarTy is a FixedVectorType, a vector will be extracted instead of a scalar.

Definition at line 254 of file SLPCostAnalysis.cpp.

References assert(), llvm::cast(), CostKind, llvm::dyn_cast(), getShuffleCost(), llvm::isa(), and llvm::TargetTransformInfo::SK_ExtractSubvector.

Referenced by slpvectorizer::BoUpSLP::getTreeCost().

◆ getVectorLoadShift()

uint64_t llvm::slpvectorizer::getVectorLoadShift ( ArrayRef< Value * > Scalars )

Returns the distance in bytes from the start of the vector load to the first load of Scalars: the leading undef lanes are covered by the vector load.

Definition at line 438 of file SLPMemoryUtils.cpp.

References llvm::ArrayRef< T >::begin(), llvm::cast(), llvm::find_if(), and llvm::IsaPred.

◆ getWidenedStridedCastCost()

InstructionCost llvm::slpvectorizer::getWidenedStridedCastCost	(	const TargetTransformInfo &	TTI,
		Type *	SrcTy,
		Type *	DstTy,
		const DataLayout &	DL,
		TTI::CastContextHint	CCH,
		TTI::TargetCostKind	CostKind )

Returns the cost of the cast between the widened strided access type and the entry vector type.

Definition at line 171 of file SLPCostAnalysis.cpp.

References llvm::cast(), CostKind, DL, and llvm::isPointerTy().

◆ getWidenedType()

Type * llvm::slpvectorizer::getWidenedType	(	Type *	ScalarTy,
		unsigned	VF )

Returns: the vector type of ScalarTy based on vectorization factor.

Definition at line 54 of file SLPTypeUtils.cpp.

References llvm::all_of(), assert(), llvm::dyn_cast(), llvm::FixedVectorType::get(), llvm::StructType::get(), llvm::ElementCount::getFixed(), getNumElements(), llvm::isUnpackedStructLiteral(), llvm::VectorType::isValidElementType(), llvm::isVectorizedTy(), llvm::map_to_vector(), llvm::toScalarizedTy(), and llvm::toVectorizedTy().

◆ hasFullVectorsOrPowerOf2()

bool llvm::slpvectorizer::hasFullVectorsOrPowerOf2	(	const TargetTransformInfo &	TTI,
		Type *	Ty,
		unsigned	Sz,
		bool	ReVec )

Returns true if widened type of Ty elements with size Sz represents full vector type, i.e.

adding extra element results in extra parts upon type legalization.

Definition at line 114 of file SLPTypeUtils.cpp.

References getWidenedType(), llvm::has_single_bit(), llvm::isa(), and isValidElementType().

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), and getNumberOfPartsOrRegs().

◆ hasOnlyAbsorbableCopyableFMulOrFAdds()

bool llvm::slpvectorizer::hasOnlyAbsorbableCopyableFMulOrFAdds ( ArrayRef< Value * > VL )

Checks if every copyable in VL is an absorbable fmul/fadd: the binops die instead of being computed and gathered.

Operand order is normalized when the operands are built.

Definition at line 606 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, llvm::isa(), isAbsorbableFMulOrFAdd(), llvm::RecurrenceDescriptor::isFMulAddIntrinsic(), and llvm::make_filter_range().

◆ inversePermutation()

void llvm::slpvectorizer::inversePermutation	(	ArrayRef< unsigned >	Indices,
		SmallVectorImpl< int > &	Mask )

Compute the inverse permutation Mask of Indices.

Definition at line 316 of file SLPUtils.cpp.

References I, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::addOrdered(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getCastContextHint(), slpvectorizer::BoUpSLP::getReorderingData(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), reorderOrder(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ isAbsorbableCopyableFMulOrFAdd()

bool llvm::slpvectorizer::isAbsorbableCopyableFMulOrFAdd	(	const InstructionsState &	S,
		Value *	V )

Checks if V is a copyable single-use fmul/fadd, absorbable as fmuladd(a, b, -0.0) or fmuladd(1.0, a, b).

Definition at line 598 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, and llvm::slpvectorizer::InstructionsState::isCopyableElement().

◆ isAbsorbableFMulOrFAdd()

bool llvm::slpvectorizer::isAbsorbableFMulOrFAdd	(	ArrayRef< Value * >	VL,
		Value *	V )

Checks if V is a single-use fmul/fadd with operands outside VL.

Definition at line 588 of file SLPCompatibilityAnalysis.cpp.

References llvm::dyn_cast(), I, and llvm::none_of().

Referenced by hasOnlyAbsorbableCopyableFMulOrFAdds().

◆ isAllowedNonPowerOf2VF()

bool llvm::slpvectorizer::isAllowedNonPowerOf2VF	(	unsigned	NumElts,
		bool	AllowNonPowerOf2 )

True when AllowNonPowerOf2 is set and NumElts is a supported non-power-of-2 width: NumElts + 1 must be a power of two (e.g.

3 or 7 lanes, i.e. almost a full power-of-2 register).

Definition at line 129 of file SLPTypeUtils.cpp.

References llvm::has_single_bit().

◆ isAlternateInstruction()

bool llvm::slpvectorizer::isAlternateInstruction	(	Instruction *	I,
		Instruction *	MainOp,
		Instruction *	AltOp,
		const TargetLibraryInfo &	TLI )

Checks if the specified instruction I is an alternate operation for the given MainOp and AltOp instructions.

Definition at line 994 of file SLPCompatibilityAnalysis.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), llvm::slpvectorizer::InstructionsState::getMatchingMainOpOrAltOp(), llvm::CmpInst::getSwappedPredicate(), I, isCmpSameOrSwapped(), and P.

Referenced by slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::getReorderingData(), and scanAltAssociativeOperands().

◆ isBinOpIdentityConstant()

bool llvm::slpvectorizer::isBinOpIdentityConstant	(	const Value *	V,
		unsigned	Opcode )

Returns: True if V is the integer identity constant for binary Opcode (e.g. 0 for add, 1 for mul, all-ones for and). Floating-point identities are excluded: a ConstantInt never matches the ConstantFP getBinOpIdentity() returns for FAdd/FMul, whose identity fast-math may break anyway.

Definition at line 44 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::ConstantExpr::getBinOpIdentity().

Referenced by getReassocColumnsQuality().

◆ isBoolBitmaskRdx()

BoolBitmask llvm::slpvectorizer::isBoolBitmaskRdx	(	RecurKind	RdxKind,
		const SmallDenseMap< Value *, NarrowedLeafInfo > &	NarrowedLeafShifts,
		const DataLayout &	DL )

Returns: the BoolBitmask match if the or-reduction of the narrowed leaves packs each boolean (0 or 1 after masking) leaf into its own bit position 0..N-1, i.e. it is a bitcast of the per-lane zero tests to an iN integer; BoolBitmask::None otherwise.

Definition at line 76 of file SLPReductionUtils.cpp.

References llvm::computeKnownBits(), DL, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::Known, NeedMask, NoMask, None, llvm::Or, llvm::SmallBitVector::set(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::size(), and llvm::SmallBitVector::test().

◆ isCmpSameOrSwapped()

bool llvm::slpvectorizer::isCmpSameOrSwapped	(	const CmpInst *	BaseCI,
		const CmpInst *	CI,
		const TargetLibraryInfo &	TLI )

static

Returns: true if a compare instruction CI has similar "look" and same predicate as BaseCI, "as is" or with its operands and predicate swapped, false otherwise.

Definition at line 725 of file SLPCompatibilityAnalysis.cpp.

References areCompatibleCmpOps(), assert(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::CmpInst::getSwappedPredicate(), and llvm::Value::getType().

Referenced by getSameOpcode(), and isAlternateInstruction().

◆ isCommutableOperand()

bool llvm::slpvectorizer::isCommutableOperand	(	const Instruction *	I,
		Value *	ValWithUses,
		unsigned	Op,
		bool	IsCopyable = false )

Checks if the operand is commutative.

In commutative operations, not all operands might commutable, e.g. for fmuladd only 2 first operands are commutable.

Definition at line 218 of file SLPUtils.cpp.

References assert(), llvm::dyn_cast(), I, llvm::isa(), and isCommutative().

Referenced by slpvectorizer::BoUpSLP::::HorizontalReduction.

◆ isCommutative() [1/2]

bool llvm::slpvectorizer::isCommutative ( const Instruction * I )

This is a helper function to check whether I is commutative.

This is a convenience wrapper that calls the two-parameter version of isCommutative with the same instruction for both parameters. This is the common case where the instruction being checked for commutativity is the same as the instruction whose uses are analyzed for special patterns (see the two-parameter version above for details).

Parameters

I	The instruction to check for commutativity

Returns: true if the instruction is commutative, false otherwise

Definition at line 216 of file SLPUtils.cpp.

References I, and isCommutative().

◆ isCommutative() [2/2]

bool llvm::slpvectorizer::isCommutative	(	const Instruction *	I,
		const Value *	ValWithUses,
		bool	IsCopyable = false )

Returns: True if I is commutative, handles CmpInst and BinaryOperator. For BinaryOperator, it also checks if ValWithUses is used in specific patterns that make it effectively commutative (like equality comparisons with zero). In most cases, users should not call this function directly (since I and ValWithUses are the same). However, when analyzing interchangeable instructions, we need to use the converted opcode along with the original uses.

Parameters

I	The instruction to check for commutativity
ValWithUses	The value whose uses are analyzed for special patterns

Definition at line 181 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::Value::hasNUsesOrMore(), llvm::Value::hasUseList(), I, llvm::Value::uses(), and UsesLimit.

Referenced by slpvectorizer::BoUpSLP::::HorizontalReduction, getNumberOfPotentiallyCommutativeOps(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getScoreAtLevelRec(), isCommutableOperand(), isCommutative(), and slpvectorizer::BoUpSLP::isProfitableToReorder().

◆ isConstant()

bool llvm::slpvectorizer::isConstant ( Value * V )

Returns: True if the value is a constant (but not globals/constant expressions).

Definition at line 40 of file SLPUtils.cpp.

References llvm::isa().

Referenced by allConstant(), areCompatibleCmpOps(), arePointersCompatible(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), isCopyableGEPAddressVector(), and isVectorLikeInstWithConstOps().

◆ isCopyableGEPAddressVector()

bool llvm::slpvectorizer::isCopyableGEPAddressVector ( ArrayRef< Value * > PointerOps )

Checks if the pointers PointerOps of the gathered loads, which are not compatible in the usual sense (some of them are constant-offset pointers, some have runtime indices), still form a cheap address vector as a copyable GEP node: a splat of the common base and a vector of indices.

The constant-offset lanes are the base itself or single-index GEPs of the base with a constant index (modeled as gep V, 0 or as matching lanes with constant indices), the other lanes are single-index GEPs of the base of the same shape, whose indices are affine in a single runtime value (the stride): optionally cast (all with the same cast opcode) values, each of which is the stride itself or a binary operation of the stride and a constant, like b[i * S] or b[S + i]. Duplicate lanes are not accepted: the node would be a shuffled non-full vector, gathering the loads is cheaper then.

Definition at line 926 of file SLPUtils.cpp.

References llvm::sampleprof::Base, llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::find_if_not(), llvm::from_range, GEP, llvm::isa(), isConstant(), P, llvm::ArrayRef< T >::size(), and llvm::SmallPtrSetImplBase::size().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ isEquivalentIntrinsicID()

Intrinsic::ID llvm::slpvectorizer::isEquivalentIntrinsicID	(	Intrinsic::ID	LHS,
		Intrinsic::ID	RHS )

Checks if LHS and RHS are the same intrinsic, or one is llvm.fma and the other is llvm.fmuladd, since both lower to the same fused vector operation.

Returns: the intrinsic ID to use for the pair (RHS if the IDs match, otherwise Intrinsic::fma), or Intrinsic::not_intrinsic if they are not equivalent.

Definition at line 172 of file SLPUtils.cpp.

References llvm::Intrinsic::not_intrinsic.

Referenced by getSameOpcode(), and llvm::slpvectorizer::InstructionsState::isSameOperation().

◆ isFixedVectorShuffle()

std::optional< TargetTransformInfo::ShuffleKind > llvm::slpvectorizer::isFixedVectorShuffle	(	ArrayRef< Value * >	VL,
		SmallVectorImpl< int > &	Mask,
		AssumptionCache *	AC )

Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.

Checks if the values in VL can be represented as a shuffle of at most two vector operands (extractelement lanes).

TODO: Can we split off and reuse the shuffle mask detection from ShuffleVectorInst/getShuffleCost?

On success, Mask is the equivalent shuffle mask.

Definition at line 601 of file SLPUtils.cpp.

References llvm::any_of(), llvm::cast(), llvm::dyn_cast(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::find_if(), getNumElements(), llvm::Value::getType(), llvm::ExtractElementInst::getVectorOperand(), I, llvm::isa(), llvm::IsaPred, llvm::isGuaranteedNotToBePoison(), isUndefVector(), llvm::make_isa_range(), llvm::PoisonMaskElem, Select, Size, llvm::ArrayRef< T >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, llvm::TargetTransformInfo::SK_PermuteTwoSrc, llvm::TargetTransformInfo::SK_Select, and llvm::Unknown.

◆ isMaskedLoadCompress() [1/2]

bool llvm::slpvectorizer::isMaskedLoadCompress	(	ArrayRef< Value * >	VL,
		ArrayRef< Value * >	PointerOps,
		ArrayRef< unsigned >	Order,
		const TargetTransformInfo &	TTI,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		AssumptionCache &	AC,
		const DominatorTree &	DT,
		const TargetLibraryInfo &	TLI,
		const TargetTransformInfo::TargetCostKind	CostKind,
		const function_ref< bool(Value *)>	AreAllUsersVectorized,
		bool	ReVec )

Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.

Definition at line 376 of file SLPMemoryUtils.cpp.

References CostKind, DL, and isMaskedLoadCompress().

◆ isMaskedLoadCompress() [2/2]

bool llvm::slpvectorizer::isMaskedLoadCompress	(	ArrayRef< Value * >	VL,
		ArrayRef< Value * >	PointerOps,
		ArrayRef< unsigned >	Order,
		const TargetTransformInfo &	TTI,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		AssumptionCache &	AC,
		const DominatorTree &	DT,
		const TargetLibraryInfo &	TLI,
		const TargetTransformInfo::TargetCostKind	CostKind,
		const function_ref< bool(Value *)>	AreAllUsersVectorized,
		bool	ReVec,
		bool &	IsMasked,
		unsigned &	InterleaveFactor,
		SmallVectorImpl< int > &	CompressMask,
		VectorType *&	LoadVecTy )

Checks if the VL can be transformed to a (masked)load + compress or (masked) interleaved load.

Definition at line 234 of file SLPMemoryUtils.cpp.

References llvm::accumulate(), assert(), llvm::ArrayRef< T >::back(), buildCompressMask(), C(), llvm::cast(), CostKind, DL, llvm::ArrayRef< T >::empty(), llvm::enumerate(), llvm::ArrayRef< T >::front(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::APInt::getAllOnes(), getFullVectorNumberOfElements(), getGEPCosts(), llvm::TargetTransformInfo::getOperandInfo(), llvm::getPointersDiff(), getScalarizationOverhead(), getShuffleCost(), getWidenedType(), I, InstructionCost, inversePermutation(), llvm::isSafeToLoadUnconditionally(), llvm::PoisonMaskElem, reorderScalars(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::seq(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::TargetTransformInfo::SK_PermuteSingleSrc, and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads(), and isMaskedLoadCompress().

◆ isMaskedStoreCompress()

bool llvm::slpvectorizer::isMaskedStoreCompress	(	ArrayRef< Value * >	VL,
		ArrayRef< Value * >	PointerOps,
		ArrayRef< unsigned >	Order,
		const TargetTransformInfo &	TTI,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		Align	CommonAlignment,
		SmallVectorImpl< int > &	ReuseShuffleIndices,
		FixedVectorType *&	StoreVecTy )

Checks if the stores VL with pointers PointerOps can be lowered as a single masked store.

On success StoreVecTy is the widened store type and ReuseShuffleIndices is the expand mask that places each stored value at its element offset from the base (poison in the gaps).

Definition at line 449 of file SLPMemoryUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::back(), llvm::cast(), llvm::TargetTransformInfo::ConstantMask, DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::front(), llvm::getPointersDiff(), getWidenedType(), I, llvm::Type::isFloatingPointTy(), llvm::Type::isIntOrPtrTy(), Off, llvm::PoisonMaskElem, llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::seq(), and llvm::ArrayRef< T >::size().

◆ isNonProfitableIndex()

bool llvm::slpvectorizer::isNonProfitableIndex ( const Instruction * I )

static

Returns true if I is a part of a single-use chain, computing an address, which does not pay off the vectorization: all the lanes are extracted for the scalar addresses, the extracts delay the memory accesses.

Definition at line 1073 of file SLPUtils.cpp.

References _, I, llvm::isa(), and llvm::seq().

Referenced by isOnceUsedSeed().

◆ isOnceUsedSeed()

bool llvm::slpvectorizer::isOnceUsedSeed ( const Instruction * I )

Returns true if I forms a vectorizable bundle on its own and its single user does not tear the vector apart.

Loads and addresses are excluded: the tree is built without the users, so it does not pay off the extracts. A cast, feeding a multi-used cast, is excluded for the same reason, such a user stays scalar. The fp-to-int conversions move the result to the other register domain, so the extracts are paid on top of the repacking. The values, feeding the inserts, are vectorized together with them by the dedicated attempt.

Definition at line 1086 of file SLPUtils.cpp.

References C(), llvm::dyn_cast(), llvm::ConstantExpr::getBinOpAbsorber(), llvm::ConstantExpr::getBinOpIdentity(), I, llvm::isa(), isNonProfitableIndex(), and llvm::seq().

◆ isReassocChainLink()

bool llvm::slpvectorizer::isReassocChainLink ( const Instruction * I )

Returns: True if I can be a link of a flattenable binary chain: subtracts flatten as adds of a negated leaf, float subtracts need reassoc to allow the regrouping.

Definition at line 75 of file SLPUtils.cpp.

References I.

Referenced by scanAltAssociativeOperands(), and scanAssociativeOperands().

◆ isReductionCandidate()

bool llvm::slpvectorizer::isReductionCandidate ( Instruction * I )

Returns: true if I is a candidate instruction for reduction vectorization.

Definition at line 61 of file SLPReductionUtils.cpp.

References I, IsSelect(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and matchRdxBop().

◆ isRepeatedNonIdentityClusteredMask()

bool llvm::slpvectorizer::isRepeatedNonIdentityClusteredMask	(	ArrayRef< int >	Mask,
		unsigned	Sz )

Checks if the given mask is a "clustered" mask with the same clusters of size Sz, which are not identity submasks.

Definition at line 398 of file SLPUtils.cpp.

References llvm::Cluster, I, and llvm::ShuffleVectorInst::isIdentityMask().

◆ isReverseOrder()

bool llvm::slpvectorizer::isReverseOrder ( ArrayRef< unsigned > Order )

Check if Order represents reverse order.

Definition at line 389 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::empty(), llvm::enumerate(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ isSelectedBaseLoad()

bool llvm::slpvectorizer::isSelectedBaseLoad	(	Type *	ScalarTy,
		ArrayRef< Value * >	PointerOps,
		const DataLayout &	DL,
		Value *&	TrueBase,
		Value *&	FalseBase,
		SmallVectorImpl< Value * > &	Conditions )

Checks if the loads with scalar type ScalarTy and pointer operands PointerOps are each (optionally via a constant-offset GEP) a select Cond, A, B picking between the same two base pointers A/B on every lane - the shape a fully unrolled x = cond ?

A[i] : B[i] takes. On success TrueBase / FalseBase are the candidate bases and Conditions holds each lane's select condition, used to build the blend mask. Lane Idx must be at Base + Idx * sizeof(ScalarTy); only dense, natural lane order starting at the base is recognized (reordered or partial groups fall back to Gather/Scatter).

Definition at line 841 of file SLPUtils.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::sampleprof::Base, DL, llvm::dyn_cast(), llvm::enumerate(), F, GEP, llvm::APInt::getZExtValue(), llvm::APInt::isNegative(), llvm::Offset, P, llvm::ArrayRef< T >::size(), and T.

Referenced by slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ isSimple()

bool llvm::slpvectorizer::isSimple ( Instruction * I )

Returns: True if the instruction is not a volatile or atomic load/store.

Definition at line 831 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, and MI.

◆ isSplat()

bool llvm::slpvectorizer::isSplat ( ArrayRef< Value * > VL )

Returns: True if all of the values in VL are identical or some of them are UndefValue.

Definition at line 158 of file SLPUtils.cpp.

References llvm::isa(), and llvm::make_filter_range().

Referenced by llvm::DOTGraphTraits< BoUpSLP * >::getNodeLabel(), slpvectorizer::BoUpSLP::getReorderingData(), and matchGatheredExtractedFields().

◆ isTypeLimitNeutral()

bool llvm::slpvectorizer::isTypeLimitNeutral	(	const Value *	V,
		bool	IsMax,
		const DataLayout &	DL )

Returns true if the type limit of the smax (IsMax) or smin lane V stays neutral after narrowing.

Definition at line 156 of file SLPTypeUtils.cpp.

References DL, llvm::isKnownNegative(), and llvm::isKnownNonNegative().

◆ isUndefTolerantBinOp()

bool llvm::slpvectorizer::isUndefTolerantBinOp ( const Value * V )

Returns: True if V is an integer binary operator, which gives an undef result for the undef operands: neither a shift nor a division.

Definition at line 49 of file SLPUtils.cpp.

References llvm::Value::getType(), llvm::Instruction::isIntDivRem(), llvm::Type::isIntOrIntVectorTy(), llvm::Instruction::isShift(), llvm::PatternMatch::m_BinOp(), and llvm::PatternMatch::match().

Referenced by getLoadOfUndefTolerantBinOp().

◆ isUndefVector()

template<bool IsPoisonOnly>

SmallBitVector llvm::slpvectorizer::isUndefVector	(	const Value *	V,
		const SmallBitVector &	UseMask = {} )

Checks if the given value is actually an undefined constant vector.

Also, if the UseMask is not empty, tries to check if the non-masked elements actually mask the insertelement buildvector, if any.

Definition at line 745 of file SLPUtils.cpp.

References llvm::sampleprof::Base, C(), llvm::dyn_cast(), getElementIndex(), I, II, llvm::isa(), isUndefVector(), llvm::SmallBitVector::reset(), and T.

Referenced by llvm::slpvectorizer::BaseShuffleAnalysis::createShuffle(), generateKeySubkey(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), isFixedVectorShuffle(), isUndefVector(), llvm::slpvectorizer::BaseShuffleAnalysis::peekThroughShuffles(), and performExtractsShuffleAction().

◆ isUndefVector< false >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< false >	(	const Value *	,
		const SmallBitVector &	)

◆ isUndefVector< true >()

template SmallBitVector llvm::slpvectorizer::isUndefVector< true >	(	const Value *	,
		const SmallBitVector &	)

Referenced by performExtractsShuffleAction().

◆ isUsedOutsideBlock()

bool llvm::slpvectorizer::isUsedOutsideBlock ( Value * V )

Checks if the provided value does not require scheduling.

It does not require scheduling if this is not an instruction or it is an instruction that does not read/write memory and all users are phi nodes or instructions from different blocks.

Definition at line 483 of file SLPUtils.cpp.

References llvm::all_of(), llvm::dyn_cast(), I, and UsesLimit.

Referenced by doesNotNeedToBeScheduled(), and doesNotNeedToSchedule().

◆ isValidElementType()

bool llvm::slpvectorizer::isValidElementType	(	Type *	Ty,
		bool	ReVec )

Predicate for the element types that the SLP vectorizer supports.

The most important thing to filter here are types which are invalid in LLVM vectors. We also filter target specific types which have absolutely no meaningful vectorization path such as x86_fp80 and ppc_f128. This just avoids spending time checking the cost model and realizing that they will be inevitably scalarized.

Definition at line 32 of file SLPTypeUtils.cpp.

References llvm::canVectorizeTy(), llvm::getVectorizedTypeVF(), llvm::isVectorizedTy(), and llvm::toScalarizedTy().

Referenced by slpvectorizer::BoUpSLP::canMapToVector(), compareCmp(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), gatherPossiblyVectorizableLoads(), getFloorFullVectorNumberOfElements(), getFullVectorNumberOfElements(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), hasFullVectorsOrPowerOf2(), isPoorThroughputOp(), tryToVectorizeSequence(), llvm::SLPVectorizerPass::vectorizeCmpInsts(), and llvm::SLPVectorizerPass::vectorizeNonVectorizableInsts().

◆ isValidForAlternation()

bool llvm::slpvectorizer::isValidForAlternation ( unsigned Opcode )

Returns: true if Opcode is allowed as part of the main/alternate instruction for SLP vectorization.

Example of unsupported opcode is SDIV that can potentially cause UB if the "shuffled out" lane would result in division by zero.

Definition at line 42 of file SLPCompatibilityAnalysis.cpp.

References llvm::Instruction::isIntDivRem().

Referenced by generateKeySubkey(), and getSameOpcode().

◆ isVectorLikeInstWithConstOps()

bool llvm::slpvectorizer::isVectorLikeInstWithConstOps ( Value * V )

Checks if V is one of vector-like instructions, i.e.

undef, insertelement/extractelement with constant indices for fixed vector type or extractvalue instruction.

Definition at line 83 of file SLPUtils.cpp.

References llvm::dyn_cast(), I, llvm::isa(), and isConstant().

Referenced by allSameBlock(), generateKeySubkey(), getSameOpcode(), llvm::slpvectorizer::InstructionsState::isCopyableElement(), llvm::slpvectorizer::InstructionsState::isNonSchedulable(), and tryToFindDuplicates().

◆ lookThroughCastRoundTrip()

Instruction * llvm::slpvectorizer::lookThroughCastRoundTrip	(	Value *	V,
		bool	MustBeElidable )

If V is a single-use fpext of a single-use fptrunc forming a round-trip back to the type of V, returns the fptrunc; the round-trip source is its operand, always an instruction of the same type as V.

If MustBeElidable, matches only when the intermediate rounding may be removed: both casts must allow contraction and the widening cast cannot produce nan/inf.

Definition at line 1113 of file SLPUtils.cpp.

References llvm::dyn_cast(), and llvm::isa().

◆ matchExtractedField()

std::optional< std::tuple< Value *, unsigned, unsigned > > llvm::slpvectorizer::matchExtractedField ( Value * V )

static

Checks if V is a zero-extended sub-field of a wider integer scalar.

Returns the source scalar, the field width and the field offset.

Definition at line 1252 of file SLPUtils.cpp.

References llvm::BitWidth, llvm::Type::getIntegerBitWidth(), llvm::APInt::getLimitedValue(), llvm::Value::getType(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_LShr(), llvm::PatternMatch::m_Shr(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExt(), llvm::PatternMatch::match(), and llvm::Offset.

Referenced by matchGatheredExtractedFields().

◆ matchGatheredExtractedFields()

std::optional< std::tuple< Value *, unsigned, SmallVector< int > > > llvm::slpvectorizer::matchGatheredExtractedFields	(	ArrayRef< Value * >	VL,
		const DataLayout &	DL )

Checks if the values in VL are zero-extended sub-fields of the same wider integer scalar.

Returns the source scalar, the field width and the field permutation mask. The extraction dual of the lane-packing layout. The field-to-lane mapping of the bitcast to the field vector is defined for little-endian targets only.

Definition at line 1340 of file SLPUtils.cpp.

References llvm::all_of(), DL, llvm::enumerate(), llvm::find_if(), llvm::First, llvm::ArrayRef< T >::front(), llvm::isa(), isSplat(), llvm::make_filter_range(), matchExtractedField(), llvm::not_equal_to(), P, llvm::PoisonMaskElem, and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::getCastContextHint(), slpvectorizer::BoUpSLP::processExtractedFieldsGather(), and slpvectorizer::BoUpSLP::transformNodes().

◆ matchPackedFields()

bool llvm::slpvectorizer::matchPackedFields	(	Value *	V,
		unsigned	MaxDepth,
		SmallVectorImpl< Value * > &	Fields,
		SmallVectorImpl< Instruction * > &	Chain )

Matches V as the zero-extended fields, placed one after another from the lowest bit and filling V completely.

The instructions are looked through up to MaxDepth. Appends the fields in the order of their positions to Fields and the instructions of the pack to Chain.

Definition at line 100 of file SLPReductionUtils.cpp.

References A(), llvm::append_range(), B(), collectNarrowedLeaves(), llvm::dyn_cast(), llvm::enumerate(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::Type::getIntegerBitWidth(), llvm::Type::isIntegerTy(), llvm::map_range(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::sort().

Referenced by vectorizePackedAggregate().

◆ matchRdxBop()

bool llvm::slpvectorizer::matchRdxBop	(	Instruction *	I,
		Value *&	V0,
		Value *&	V1 )

static

Definition at line 31 of file SLPReductionUtils.cpp.

References I, llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_FMaximum(), llvm::PatternMatch::m_FMaxNum(), llvm::PatternMatch::m_FMinimum(), llvm::PatternMatch::m_FMinNum(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::V1.

Referenced by getNonPhiOperand(), and isReductionCandidate().

◆ redirectDbgValues()

void llvm::slpvectorizer::redirectDbgValues	(	Instruction &	From,
		Value &	To )

The debug values of the erased From are kept on its replacement To.

A record placed before To is cloned right after it, while the original one is killed together with the scalar, so the variable is undefined up to To. The clone is skipped if it would pass a record of the same variable, otherwise the variable would show a stale value.

Definition at line 1531 of file SLPUtils.cpp.

References llvm::any_of(), llvm::DbgVariableRecord::clone(), llvm::dyn_cast(), llvm::filterDbgVars(), llvm::findDbgValues(), llvm::Instruction::getDbgRecordRange(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), I, llvm::make_range(), llvm::Other, and llvm::DbgVariableRecord::replaceVariableLocationOp().

Referenced by slpvectorizer::BoUpSLP::vectorizeTree().

◆ reorderOrder()

void llvm::slpvectorizer::reorderOrder	(	SmallVectorImpl< unsigned > &	Order,
		ArrayRef< int >	Mask,
		bool	BottomOrder = false )

Reorders the given Order according to the given Mask.

Order - is the original order of the scalars. Procedure transforms the provided order in accordance with the given Mask. If the resulting Order is just an identity order, Order is cleared.

Definition at line 345 of file SLPUtils.cpp.

References llvm::all_of(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::Data, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), fixupOrderingIndices(), I, inversePermutation(), llvm::ShuffleVectorInst::isIdentityMask(), llvm::PoisonMaskElem, reorderReuses(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::getReorderingData(), slpvectorizer::BoUpSLP::reorderBottomToTop(), and slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ reorderReuses()

void llvm::slpvectorizer::reorderReuses	(	SmallVectorImpl< int > &	Reuses,
		ArrayRef< int >	Mask )

Reorders the given Reuses mask according to the given Mask.

Reuses contains original mask for the scalars reused in the node. Procedure transform this mask in accordance with the given Mask.

Definition at line 335 of file SLPUtils.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::reorderBottomToTop(), and reorderOrder().

◆ reorderScalars()

void llvm::slpvectorizer::reorderScalars	(	SmallVectorImpl< Value * > &	Scalars,
		ArrayRef< int >	Mask )

Reorders the list of scalars in accordance with the given Mask.

Definition at line 325 of file SLPUtils.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::PoisonValue::get(), I, llvm::PoisonMaskElem, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::SmallVectorImpl< T >::swap().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::adjustExtracts(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::adjustExtracts(), isMaskedLoadCompress(), slpvectorizer::BoUpSLP::processBuildVector(), slpvectorizer::BoUpSLP::reorderBottomToTop(), slpvectorizer::BoUpSLP::reorderTopToBottom(), and slpvectorizer::BoUpSLP::transformNodes().

◆ replicateMask()

SmallVector< Constant * > llvm::slpvectorizer::replicateMask	(	ArrayRef< Constant * >	Val,
		unsigned	VF )

Replicates the given Val VF times.

Definition at line 1046 of file SLPUtils.cpp.

References assert(), C(), llvm::enumerate(), I, llvm::none_of(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather().

◆ scanAltAssociativeOperands()

SmallVector< SmallVector< Value * > > llvm::slpvectorizer::scanAltAssociativeOperands	(	const InstructionsState &	S,
		const TargetLibraryInfo &	TLI,
		ArrayRef< Value * >	VL,
		ArrayRef< Value * >	Op0,
		ArrayRef< Value * >	Op1,
		SmallVectorImpl< Value * > &	ReassocScalars,
		SmallBitVector &	SubLanes )

Peel the per-lane associative chains of an alternate node into operand columns.

Lanes peel in lockstep and only chain links with the lane's own opcode, so every combine level keeps the root's main/alt opcode pattern and a subtract lane never becomes an add of a negated leaf. Only the leading (running) column peels: peeling a subtracted subtract would flip signs. SubLanes records the subtract lanes for the realignment sign query. Returns the flattened columns, empty when no level peels.

Definition at line 1017 of file SLPCompatibilityAnalysis.cpp.

References llvm::all_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::slpvectorizer::InstructionsState::getAltOp(), llvm::slpvectorizer::InstructionsState::getAltOpcode(), llvm::slpvectorizer::InstructionsState::getMainOp(), llvm::slpvectorizer::InstructionsState::getOpcode(), llvm::User::getOperand(), I, llvm::SmallVectorImpl< T >::insert(), isAlternateInstruction(), llvm::slpvectorizer::InstructionsState::isAltShuffle(), llvm::Instruction::isCommutative(), isReassocChainLink(), llvm::map_to_vector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallBitVector::resize(), llvm::seq(), llvm::SmallBitVector::set(), llvm::ArrayRef< T >::size(), and std::swap().

◆ shortBundleName()

std::string llvm::slpvectorizer::shortBundleName	(	ArrayRef< Value * >	VL,
		int	Idx )

Print a short descriptor of the instruction bundle suitable for debug output.

Definition at line 120 of file SLPUtils.cpp.

References llvm::ArrayRef< T >::front(), and llvm::ArrayRef< T >::size().

Referenced by slpvectorizer::BoUpSLP::processBuildVector().

◆ transformScalarShuffleIndiciesToVector()

void llvm::slpvectorizer::transformScalarShuffleIndiciesToVector	(	unsigned	VecTyNumElements,
		SmallVectorImpl< int > &	Mask )

Definition at line 506 of file SLPUtils.cpp.

References llvm::enumerate(), I, llvm::MutableArrayRef(), llvm::PoisonMaskElem, and llvm::seq().

Referenced by llvm::slpvectorizer::BaseShuffleAnalysis::createShuffle().

◆ tryEmitBoolReduxBitcastCmp()

Value * llvm::slpvectorizer::tryEmitBoolReduxBitcastCmp	(	IRBuilderBase &	Builder,
		const TargetTransformInfo &	TTI,
		RecurKind	RdxKind,
		Value *	Vec,
		const Value *	Root,
		FastMathFlags	FMF,
		TargetTransformInfo::TargetCostKind	CostKind )

Emits the booleanized logical and/or reduction of Vec with the i1 root Root as trunc+bitcast+cmp (all-ones comparison for And, zero for Or) if it is cheaper than the wide reduction plus the result trunc.

Returns: the i1 result or nullptr if the wide reduction form is cheaper.

Definition at line 127 of file SLPReductionUtils.cpp.

References llvm::And, llvm::cast(), CostKind, DL, llvm::dyn_cast(), llvm::FixedVectorType::get(), llvm::Constant::getAllOnesValue(), getBoolReduxWideRdxCost(), getDebugLoc(), llvm::Constant::getNullValue(), llvm::Value::getType(), I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, InstructionCost, and T.

Variable Documentation

◆ MaxBitPackAnalysisDepth

unsigned llvm::slpvectorizer::MaxBitPackAnalysisDepth = MaxAnalysisRecursionDepth + 2

constexpr

Deeper than the standard analysis recursion depth to keep the numeric bound precise through arithmetic carry chains.

Definition at line 1381 of file SLPUtils.cpp.

Referenced by getScalarMaxValue().

◆ UsesLimit

int llvm::slpvectorizer::UsesLimit = 64

inlineconstexpr

Limit of the number of uses for potentially transformed instructions/values, used in checks to avoid compile-time explode.

Definition at line 51 of file SLPUtils.h.

Referenced by slpvectorizer::BoUpSLP::buildExternalUses(), slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), slpvectorizer::BoUpSLP::getTreeCost(), isCommutative(), and isUsedOutsideBlock().

Classes

Enumerations

Functions

Variables

Detailed Description

Enumeration Type Documentation

◆ BoolBitmask

◆ UseMask

Function Documentation

◆ addMask()

◆ allConstant()

◆ allSameBlock()

◆ allSameOpcode()

◆ allSameType()

◆ areAllOperandsNonInsts()

◆ areCompatibleCmpOps()

◆ arePointersCompatible()

◆ buildBitPack()

◆ buildCompressMask()

◆ buildUseMask()

◆ calculateRtStride()

◆ calculateShufflevectorMask()

◆ canWidenLoadsOverUndefLanes()

◆ clusterSortPtrAccesses()

◆ collectNarrowedLeaves()

◆ collectNarrowedLeavesImpl()

◆ combineOrders()

◆ computeBitPackInfo()

◆ computeBlendedLoadBaseAlignment()

◆ computeCommonAlignment()

◆ computeCommonAlignment< LoadInst >()

◆ computeCommonAlignment< StoreInst >()

◆ convertTo()

◆ createExtractVector()

◆ createInsertVector()

◆ createWidenedStridedCast()

◆ doesInTreeUserNeedToExtract()

◆ doesNotNeedToBeScheduled()

◆ doesNotNeedToSchedule()

◆ findInstructionWithOpcode()

◆ fixupOrderingIndices()

◆ getAltInstrMask()

◆ getBitPackCost()

◆ getBitPackMask()

◆ getBlendedLoadCost()

◆ getBoolBitmaskCost()

◆ getBoolLogicRdxBitcastCost()

◆ getBoolReduxBitcastCmpCost()

◆ getBoolReduxResultCCH()

◆ getBoolReduxWideRdxCost()

◆ getBoolReduxWideTy()

◆ getCmpBoundaryFamily()

◆ getCommonGEPIndexType()

◆ getElementIndex()

◆ getExtractIndex()

◆ getExtractWithExtendCost()

◆ getFloorFullVectorNumberOfElements()

◆ getFullVectorNumberOfElements()

◆ getGEPCosts()

◆ getI1ReductionCost()

◆ getInsertExtractIndex()

◆ getInsertExtractIndex< ExtractElementInst >()

◆ getInsertExtractIndex< InsertElementInst >()

◆ getLoadOfUndefTolerantBinOp()

◆ getLocation()

◆ getMaskedDivRemCost()

◆ getMaskedDivRemIntrinsic()

◆ getMaskedDivRemType()

◆ getNarrowedLeafOpsCost()

◆ getNonPhiOperand()

◆ getNumberOfPartsOrRegs()

◆ getNumberOfPotentiallyCommutativeOps()

◆ getNumElements()

◆ getNumElems()

◆ getPartNumElems()

◆ getReassocCombineOpcode()

◆ getSameOpcode()

◆ getScalarizationOverhead()

◆ getScalarMaxValue()

◆ getShuffleCost()