LoopVectorize.cpp File Reference

#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanHCFGBuilder.h"
#include "VPlanTransforms.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	llvm::InnerLoopVectorizer
	InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...
class	llvm::InnerLoopUnroller
struct	llvm::EpilogueLoopVectorizationInfo
	Encapsulate information regarding vectorization of a loop and its epilogue. More...
class	llvm::InnerLoopAndEpilogueVectorizer
	An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its epilogue (residual) also vectorized. More...
class	llvm::EpilogueVectorizerMainLoop
	A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the process of vectorizing loops and their epilogues. More...
class	llvm::EpilogueVectorizerEpilogueLoop
class	llvm::LoopVectorizationCostModel
	LoopVectorizationCostModel - estimates the expected speedups due to vectorization. More...
struct	llvm::LoopVectorizationCostModel::RegisterUsage
	A struct that represents some properties of the register usage of a loop. More...

Namespaces
namespace	PreferPredicateTy
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	LV_NAME   "loop-vectorize"
#define	DEBUG_TYPE   LV_NAME

Typedefs
using	llvm::InstructionVFPair = std::pair< Instruction *, ElementCount >

Enumerations
enum	PreferPredicateTy::Option { PreferPredicateTy::ScalarEpilogue = 0 , PreferPredicateTy::PredicateElseScalarEpilogue , PreferPredicateTy::PredicateOrDontVectorize }
enum	llvm::ScalarEpilogueLowering {   llvm::CM_ScalarEpilogueAllowed , llvm::CM_ScalarEpilogueNotAllowedOptSize , llvm::CM_ScalarEpilogueNotAllowedLowTripLoop , llvm::CM_ScalarEpilogueNotNeededUsePredicate ,   llvm::CM_ScalarEpilogueNotAllowedUsePredicate }

Functions
	STATISTIC (LoopsVectorized, "Number of loops vectorized")
	STATISTIC (LoopsAnalyzed, "Number of loops analyzed for vectorization")
	STATISTIC (LoopsEpilogueVectorized, "Number of epilogues vectorized")
static bool	hasIrregularType (Type *Ty, const DataLayout &DL)
	A helper function that returns true if the given type is irregular.
static unsigned	getReciprocalPredBlockProb ()
	A helper function that returns the reciprocal of the block probability of predicated blocks.
static Constant *	getSignedIntOrFpConstant (Type *Ty, int64_t C)
	A helper function that returns an integer or floating-point constant with value C.
static std::optional< unsigned >	getSmallBestKnownTC (ScalarEvolution &SE, Loop *L)
	Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known.
static Instruction *	getDebugLocFromInstOrOperands (Instruction *I)
	Look for a meaningful debug location on the instruction or it's operands.
static void	debugVectorizationMessage (const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
	Write a DebugMsg about vectorization to the debug output stream.
static OptimizationRemarkAnalysis	createLVAnalysis (const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I)
	Create an analysis remark that explains why vectorization failed.
Value *	llvm::createStepForVF (IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
	Return a value for Step multiplied by VF.
Value *	llvm::getRuntimeVF (IRBuilderBase &B, Type *Ty, ElementCount VF)
	Return the runtime value for VF.
const SCEV *	llvm::createTripCountSCEV (Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
static Value *	llvm::getRuntimeVFAsFloat (IRBuilderBase &B, Type *FTy, ElementCount VF)
void	llvm::reportVectorizationFailure (const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
	Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding optimization remark RemarkName.
void	llvm::reportVectorizationInfo (const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
	Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
static std::string	getDebugLocString (const Loop *L)
static bool	useActiveLaneMask (TailFoldingStyle Style)
static bool	useActiveLaneMaskForControlFlow (TailFoldingStyle Style)
static bool	isExplicitVecOuterLoop (Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static void	collectSupportedLoops (Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static Value *	getStepVector (Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
	This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...) to each vector element of Val.
static void	buildScalarSteps (Value *ScalarIV, Value *Step, const InductionDescriptor &ID, VPValue *Def, VPTransformState &State)
	Compute scalar induction steps.
static Value *	emitTransformedIndex (IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, const InductionDescriptor &ID)
	Compute the transformed value of Index at offset StartValue using step StepValue.
std::optional< unsigned >	getMaxVScale (const Function &F, const TargetTransformInfo &TTI)
static bool	isIndvarOverflowCheckKnownFalse (const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
	For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.
static bool	useMaskedInterleavedAccesses (const TargetTransformInfo &TTI)
static Value *	getExpandedStep (const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
	Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static void	cse (BasicBlock *BB)
	Perform cse of induction variable instructions.
static Type *	MaybeVectorizeType (Type *Elt, ElementCount VF)
static Type *	smallestIntegerVectorType (Type *T1, Type *T2)
static Type *	largestIntegerVectorType (Type *T1, Type *T2)
static std::optional< unsigned >	getVScaleForTuning (const Loop *L, const TargetTransformInfo &TTI)
	Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range.max or otherwise returns the value returned by the corresponding TTI method.
static void	emitInvalidCostRemarks (SmallVector< InstructionVFPair > InvalidCosts, OptimizationRemarkEmitter *ORE, Loop *TheLoop)
static const SCEV *	getAddressAccessSCEV (Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
	Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.
static unsigned	determineVPlanVF (const unsigned WidestVectorRegBits, LoopVectorizationCostModel &CM)
static void	AddRuntimeUnrollDisableMetaData (Loop *L)
static VPWidenIntOrFpInductionRecipe *	createWidenInductionRecipes (PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop, VFRange &Range)
	Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static void	addCanonicalIVRecipes (VPlan &Plan, Type *IdxTy, DebugLoc DL, TailFoldingStyle Style)
static void	addUsersInExitBlock (VPBasicBlock *HeaderVPBB, VPBasicBlock *MiddleVPBB, Loop *OrigLoop, VPlan &Plan)
static ScalarEpilogueLowering	getScalarEpilogueLowering (Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static bool	processLoopInVPlanNativePath (Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static void	checkMixedPrecision (Loop *L, OptimizationRemarkEmitter *ORE)
static bool	areRuntimeChecksProfitable (GeneratedRTChecks &Checks, VectorizationFactor &VF, std::optional< unsigned > VScale, Loop *L, ScalarEvolution &SE)

Variables
const char	VerboseDebug [] = DEBUG_TYPE "-verbose"
static cl::opt< bool >	EnableEpilogueVectorization ("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static cl::opt< unsigned >	EpilogueVectorizationForceVF ("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static cl::opt< unsigned >	EpilogueVectorizationMinVF ("epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned >	TinyTripCountVectorThreshold ("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
	Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.
static cl::opt< unsigned >	VectorizeMemoryCheckThreshold ("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static cl::opt< PreferPredicateTy::Option >	PreferPredicateOverEpilogue ("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< TailFoldingStyle >	ForceTailFoldingStyle ("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check")))
static cl::opt< bool >	MaximizeBandwidth ("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static cl::opt< bool >	EnableInterleavedMemAccesses ("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool >	EnableMaskedInterleavedMemAccesses ("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
	An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.
static cl::opt< unsigned >	TinyTripCountInterleaveThreshold ("tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden, cl::desc("We don't interleave loops with a estimated constant trip count " "below this number"))
static cl::opt< unsigned >	ForceTargetNumScalarRegs ("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< unsigned >	ForceTargetNumVectorRegs ("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static cl::opt< unsigned >	ForceTargetMaxScalarInterleaveFactor ("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static cl::opt< unsigned >	ForceTargetMaxVectorInterleaveFactor ("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static cl::opt< unsigned >	ForceTargetInstructionCost ("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
static cl::opt< bool >	ForceTargetSupportsScalableVectors ("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static cl::opt< unsigned >	SmallLoopCost ("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static cl::opt< bool >	LoopVectorizeWithBlockFrequency ("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static cl::opt< bool >	EnableLoadStoreRuntimeInterleave ("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool >	InterleaveSmallLoopScalarReduction ("interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, cl::desc("Enable interleaving for loops with small iteration counts that " "contain scalar reductions to expose ILP."))
	Interleave small loops with scalar reductions.
static cl::opt< unsigned >	NumberOfStoresToPredicate ("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
	The number of stores in a loop that are allowed to need predication.
static cl::opt< bool >	EnableIndVarRegisterHeur ("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< bool >	EnableCondStoresVectorization ("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static cl::opt< unsigned >	MaxNestedScalarReductionIC ("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< bool >	PreferInLoopReductions ("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static cl::opt< bool >	ForceOrderedReductions ("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static cl::opt< bool >	PreferPredicatedReductionSelect ("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
cl::opt< bool >	llvm::EnableVPlanNativePath ("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
static cl::opt< bool >	VPlanBuildStressTest ("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static cl::opt< bool >	PrintVPlansInDotFormat ("vplan-print-in-dot-format", cl::Hidden, cl::desc("Use dot format instead of plain text when dumping VPlans"))
static cl::opt< cl::boolOrDefault >	ForceSafeDivisor ("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
const char	LLVMLoopVectorizeFollowupAll [] = "llvm.loop.vectorize.followup_all"
const char	LLVMLoopVectorizeFollowupVectorized []
const char	LLVMLoopVectorizeFollowupEpilogue []

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   LV_NAME

Definition at line 156 of file LoopVectorize.cpp.

LV_NAME

#define LV_NAME   "loop-vectorize"

Definition at line 155 of file LoopVectorize.cpp.

Function Documentation

addCanonicalIVRecipes()

static void addCanonicalIVRecipes ( VPlan &  Plan, Type *  IdxTy, DebugLoc  DL, TailFoldingStyle  Style  )

static

Definition at line 8701 of file LoopVectorize.cpp.

References llvm::VPInstruction::ActiveLaneMask, llvm::VPUser::addOperand(), llvm::VPBasicBlock::appendRecipe(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::BranchOnCount, llvm::VPInstruction::CalculateTripCountMinusVF, llvm::VPInstruction::CanonicalIVIncrement, llvm::VPInstruction::CanonicalIVIncrementForPart, llvm::VPInstruction::CanonicalIVIncrementForPartNUW, llvm::VPInstruction::CanonicalIVIncrementNUW, llvm::DataAndControlFlowWithoutRuntimeCheck, DL, llvm::ConstantInt::get(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVPValueOrAddLiveIn(), llvm::None, llvm::VPInstruction::Not, and useActiveLaneMaskForControlFlow().

AddRuntimeUnrollDisableMetaData()

static void AddRuntimeUnrollDisableMetaData ( Loop *  L )

static

Definition at line 7617 of file LoopVectorize.cpp.

References Context, llvm::MDNode::get(), llvm::MDString::get(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MDNode::replaceOperandWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), and llvm::LoopVectorizePass::processLoop().

addUsersInExitBlock()

static void addUsersInExitBlock ( VPBasicBlock *  HeaderVPBB, VPBasicBlock *  MiddleVPBB, Loop *  OrigLoop, VPlan &  Plan  )

static

Definition at line 8808 of file LoopVectorize.cpp.

References llvm::VPlan::addLiveOut(), llvm::LoopBase< BlockT, LoopT >::getExitingBlock(), llvm::BasicBlock::getSinglePredecessor(), llvm::LoopBase< BlockT, LoopT >::getUniqueExitBlock(), llvm::VPlan::getVPValueOrAddLiveIn(), and llvm::BasicBlock::phis().

areRuntimeChecksProfitable()

static bool areRuntimeChecksProfitable ( GeneratedRTChecks &  Checks, VectorizationFactor &  VF, std::optional< unsigned >  VScale, Loop *  L, ScalarEvolution &  SE  )

static

Definition at line 9953 of file LoopVectorize.cpp.

References llvm::alignTo(), llvm::VectorizationFactor::Cost, llvm::dbgs(), llvm::ElementCount::getFixed(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getSmallBestKnownTC(), llvm::InstructionCost::getValue(), llvm::details::FixedOrScalableQuantity< ElementCount, unsigned >::isKnownLT(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::ElementCount::isScalar(), llvm::InstructionCost::isValid(), LLVM_DEBUG, llvm::VectorizationFactor::MinProfitableTripCount, llvm::VectorizationFactor::ScalarCost, VectorizeMemoryCheckThreshold, and llvm::VectorizationFactor::Width.

Referenced by llvm::LoopVectorizePass::processLoop().

buildScalarSteps()

static void buildScalarSteps ( Value *  ScalarIV, Value *  Step, const InductionDescriptor &  ID, VPValue *  Def, VPTransformState &  State  )

static

Compute scalar induction steps.

ScalarIV is the scalar induction variable on which to base the steps, Step is the size of the step.

Definition at line 2312 of file LoopVectorize.cpp.

References llvm::Add, assert(), Builder, llvm::VPTransformState::Builder, llvm::createStepForVF(), llvm::IRBuilderBase::CreateTrunc(), llvm::IntegerType::get(), llvm::VectorType::get(), llvm::Type::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::Type::getScalarSizeInBits(), llvm::Type::getScalarType(), getSignedIntOrFpConstant(), llvm::Value::getType(), llvm::VPTransformState::Instance, llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::Mul, llvm::vputils::onlyFirstLaneUsed(), llvm::VPTransformState::set(), llvm::VPTransformState::UF, and llvm::VPTransformState::VF.

Referenced by llvm::VPScalarIVStepsRecipe::execute().

checkMixedPrecision()

static void checkMixedPrecision ( Loop *  L, OptimizationRemarkEmitter *  ORE  )

static

Definition at line 9912 of file LoopVectorize.cpp.

References llvm::OptimizationRemarkEmitter::emit(), llvm::SmallVectorBase< Size_T >::empty(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), LV_NAME, llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::LoopVectorizePass::processLoop().

collectSupportedLoops()

static void collectSupportedLoops ( Loop &  L, LoopInfo *  LI, OptimizationRemarkEmitter *  ORE, SmallVectorImpl< Loop * > &  V  )

static

Definition at line 2208 of file LoopVectorize.cpp.

References collectSupportedLoops(), llvm::EnableVPlanNativePath, isExplicitVecOuterLoop(), llvm::LoopBlocksRPO::perform(), and VPlanBuildStressTest.

Referenced by collectSupportedLoops(), and llvm::LoopVectorizePass::runImpl().

createLVAnalysis()

static OptimizationRemarkAnalysis createLVAnalysis ( const char *  PassName, StringRef  RemarkName, Loop *  TheLoop, Instruction *  I  )

static

Create an analysis remark that explains why vectorization failed.

PassName is the name of the pass (e.g. can be AlwaysPrint). RemarkName is the identifier for the remark. If I is passed it is an instruction that prevents vectorization. Otherwise TheLoop is used for the location of the remark.

Returns

the remark object that can be streamed to.

Definition at line 968 of file LoopVectorize.cpp.

References DL, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getStartLoc(), I, and PassName.

Referenced by llvm::reportVectorizationFailure(), and llvm::reportVectorizationInfo().

createWidenInductionRecipes()

static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes ( PHINode *  Phi, Instruction *  PhiOrTrunc, VPValue *  Start, const InductionDescriptor &  IndDesc, VPlan &  Plan, ScalarEvolution &  SE, Loop &  OrigLoop, VFRange &  Range  )

static

Creates a VPWidenIntOrFpInductionRecpipe for Phi.

If needed, it will also insert a recipe to expand the step for the induction recipe.

Definition at line 8241 of file LoopVectorize.cpp.

References assert(), llvm::PHINode::getIncomingValueForBlock(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::InductionDescriptor::getStartValue(), llvm::InductionDescriptor::getStep(), and llvm::ScalarEvolution::isLoopInvariant().

cse()

static void cse ( BasicBlock *  BB )

static

Perform cse of induction variable instructions.

Definition at line 3388 of file LoopVectorize.cpp.

References llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), and llvm::make_early_inc_range().

Referenced by llvm::InnerLoopVectorizer::fixVectorizedLoop().

debugVectorizationMessage()

static void debugVectorizationMessage ( const StringRef  Prefix, const StringRef  DebugMsg, Instruction *  I  )

static

Write a DebugMsg about vectorization to the debug output stream.

If I is passed, the message relates to that particular instruction.

Definition at line 949 of file LoopVectorize.cpp.

References llvm::dbgs(), and I.

Referenced by llvm::reportVectorizationFailure(), and llvm::reportVectorizationInfo().

determineVPlanVF()

static unsigned determineVPlanVF ( const unsigned  WidestVectorRegBits, LoopVectorizationCostModel &  CM  )

static

Definition at line 7465 of file LoopVectorize.cpp.

References llvm::LoopVectorizationCostModel::getSmallestAndWidestTypes().

Referenced by llvm::LoopVectorizationPlanner::planInVPlanNativePath().

emitInvalidCostRemarks()

static void emitInvalidCostRemarks ( SmallVector< InstructionVFPair >  InvalidCosts, OptimizationRemarkEmitter *  ORE, Loop *  TheLoop  )

static

Definition at line 5371 of file LoopVectorize.cpp.

References A, assert(), B, llvm::SmallVectorBase< Size_T >::empty(), llvm::raw_ostream::flush(), I, if(), OS, llvm::reportVectorizationInfo(), llvm::sort(), and llvm::CallingConv::Tail.

emitTransformedIndex()

static Value * emitTransformedIndex ( IRBuilderBase &  B, Value *  Index, Value *  StartValue, Value *  Step, const InductionDescriptor &  ID  )

static

Compute the transformed value of Index at offset StartValue using step StepValue.

For integer induction, returns StartValue + Index * StepValue. For pointer induction, returns StartValue[Index * StepValue]. FIXME: The newly created binary instructions should contain nsw/nuw flags, which can be found from the original scalar operations.

Definition at line 2405 of file LoopVectorize.cpp.

References assert(), B, CreateAdd(), CreateMul(), llvm::Value::getName(), llvm::Value::getType(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm_unreachable, llvm::Offset, llvm::Value::setName(), X, and Y.

Referenced by llvm::InnerLoopVectorizer::createInductionResumeValue(), llvm::VPWidenPointerInductionRecipe::execute(), llvm::VPDerivedIVRecipe::execute(), and llvm::InnerLoopVectorizer::fixupIVUsers().

getAddressAccessSCEV()

static const SCEV * getAddressAccessSCEV ( Value *  Ptr, LoopVectorizationLegality *  Legal, PredicatedScalarEvolution &  PSE, const Loop *  TheLoop  )

static

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.

This SCEV can be sent to the Target in order to estimate the address calculation cost.

Definition at line 6390 of file LoopVectorize.cpp.

References llvm::ScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSE(), llvm::ScalarEvolution::isLoopInvariant(), llvm::IRSimilarity::Legal, and Ptr.

getDebugLocFromInstOrOperands()

static Instruction * getDebugLocFromInstOrOperands ( Instruction *  I )

static

Look for a meaningful debug location on the instruction or it's operands.

Definition at line 929 of file LoopVectorize.cpp.

References llvm::Empty, and I.

getDebugLocString()

static std::string getDebugLocString ( const Loop *  L )

static

Returns

string containing a file name and a line # for the given loop.

Definition at line 1040 of file LoopVectorize.cpp.

References llvm::raw_ostream::flush(), and OS.

Referenced by llvm::LoopVectorizePass::processLoop().

getExpandedStep()

static Value * getExpandedStep ( const InductionDescriptor &  ID, const SCEV2ValueTy &  ExpandedSCEVs  )

static

Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.

Definition at line 3151 of file LoopVectorize.cpp.

References assert(), llvm::CallingConv::C, and I.

Referenced by llvm::InnerLoopVectorizer::createInductionResumeValues(), and llvm::LoopVectorizePass::processLoop().

getMaxVScale()

std::optional< unsigned > getMaxVScale	(	const Function &	F,
		const TargetTransformInfo &	TTI
	)

Definition at line 2485 of file LoopVectorize.cpp.

References F, and llvm::TargetTransformInfo::getMaxVScale().

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), and isIndvarOverflowCheckKnownFalse().

getReciprocalPredBlockProb()

static unsigned getReciprocalPredBlockProb ( )

static

A helper function that returns the reciprocal of the block probability of predicated blocks.

If we return X, we are assuming the predicated block will execute once for every X iterations of the loop header.

TODO: We should use actual block probability here, if available. Currently, we always assume predicated blocks have a 50% chance of executing.

Definition at line 409 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationCostModel::expectedCost(), and llvm::LoopVectorizationCostModel::getDivRemSpeculationCost().

getScalarEpilogueLowering()

static ScalarEpilogueLowering getScalarEpilogueLowering ( Function *  F, Loop *  L, LoopVectorizeHints &  Hints, ProfileSummaryInfo *  PSI, BlockFrequencyInfo *  BFI, TargetTransformInfo *  TTI, TargetLibraryInfo *  TLI, LoopVectorizationLegality &  LVL, InterleavedAccessInfo *  IAI  )

static

Definition at line 9732 of file LoopVectorize.cpp.

References llvm::CM_ScalarEpilogueAllowed, llvm::CM_ScalarEpilogueNotAllowedOptSize, llvm::CM_ScalarEpilogueNotAllowedUsePredicate, llvm::CM_ScalarEpilogueNotNeededUsePredicate, F, llvm::LoopVectorizeHints::FK_Disabled, llvm::LoopVectorizeHints::FK_Enabled, llvm::LoopVectorizeHints::getForce(), llvm::LoopVectorizeHints::getPredicate(), llvm::IRPass, PreferPredicateTy::PredicateElseScalarEpilogue, PreferPredicateTy::PredicateOrDontVectorize, PreferPredicateOverEpilogue, llvm::TargetTransformInfo::preferPredicateOverEpilogue(), PreferPredicateTy::ScalarEpilogue, and llvm::shouldOptimizeForSize().

Referenced by llvm::LoopVectorizePass::processLoop(), and processLoopInVPlanNativePath().

getSignedIntOrFpConstant()

static Constant * getSignedIntOrFpConstant ( Type *  Ty, int64_t  C  )

static

A helper function that returns an integer or floating-point constant with value C.

Definition at line 413 of file LoopVectorize.cpp.

References llvm::CallingConv::C, llvm::ConstantFP::get(), llvm::ConstantInt::getSigned(), and llvm::Type::isIntegerTy().

Referenced by buildScalarSteps(), and llvm::VPWidenIntOrFpInductionRecipe::execute().

getSmallBestKnownTC()

static std::optional< unsigned > getSmallBestKnownTC ( ScalarEvolution &  SE, Loop *  L  )

static

Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known.

2) Returns expected trip count according to profile data if any. 3) Returns upper bound estimate if it is known. 4) Returns std::nullopt if all of the above failed.

Definition at line 424 of file LoopVectorize.cpp.

References llvm::getLoopEstimatedTripCount(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSmallConstantTripCount(), and LoopVectorizeWithBlockFrequency.

Referenced by areRuntimeChecksProfitable(), llvm::LoopVectorizePass::processLoop(), and llvm::LoopVectorizationCostModel::selectInterleaveCount().

getStepVector()

static Value * getStepVector ( Value *  Val, Value *  StartIdx, Value *  Step, Instruction::BinaryOps  BinOp, ElementCount  VF, IRBuilderBase &  Builder  )

static

This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...) to each vector element of Val.

The sequence starts at StartIndex. Opcode is relevant for FP induction variable.

Definition at line 2261 of file LoopVectorize.cpp.

References assert(), Builder, llvm::IntegerType::get(), llvm::VectorType::get(), llvm::Type::getContext(), llvm::Type::getScalarSizeInBits(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), and llvm::ElementCount::isVector().

Referenced by llvm::VPWidenIntOrFpInductionRecipe::execute().

getVScaleForTuning()

static std::optional< unsigned > getVScaleForTuning ( const Loop *  L, const TargetTransformInfo &  TTI  )

static

Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range.max or otherwise returns the value returned by the corresponding TTI method.

Definition at line 5307 of file LoopVectorize.cpp.

References llvm::Function::getFnAttribute(), llvm::TargetTransformInfo::getVScaleForTuning(), llvm::Attribute::getVScaleRangeMin(), and llvm::Function::hasFnAttribute().

Referenced by llvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable(), llvm::LoopVectorizePass::processLoop(), and llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

hasIrregularType()

static bool hasIrregularType ( Type *  Ty, const DataLayout &  DL  )

static

A helper function that returns true if the given type is irregular.

The type is irregular if its allocated size doesn't equal the store size of an element of the corresponding vector type.

Definition at line 396 of file LoopVectorize.cpp.

References DL.

Referenced by llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), and llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened().

isExplicitVecOuterLoop()

static bool isExplicitVecOuterLoop ( Loop *  OuterLp, OptimizationRemarkEmitter *  ORE  )

static

Definition at line 2180 of file LoopVectorize.cpp.

References llvm::LoopVectorizeHints::allowVectorization(), assert(), llvm::dbgs(), llvm::LoopVectorizeHints::emitRemarkWithHints(), llvm::LoopVectorizeHints::FK_Undefined, llvm::LoopVectorizeHints::getForce(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizeHints::getInterleave(), llvm::BasicBlock::getParent(), llvm::LoopBase< BlockT, LoopT >::isInnermost(), and LLVM_DEBUG.

Referenced by collectSupportedLoops().

isIndvarOverflowCheckKnownFalse()

static bool isIndvarOverflowCheckKnownFalse ( const LoopVectorizationCostModel *  Cost, ElementCount  VF, std::optional< unsigned >  UF = std::nullopt  )

static

For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.

If not, then we know a runtime overflow check always evaluates to false and can be removed.

Definition at line 2500 of file LoopVectorize.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getMaxVScale(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced by llvm::InnerLoopVectorizer::emitIterationCountCheck().

largestIntegerVectorType()

static Type * largestIntegerVectorType ( Type *  T1, Type *  T2  )

static

Definition at line 3514 of file LoopVectorize.cpp.

References T1.

MaybeVectorizeType()

static Type * MaybeVectorizeType ( Type *  Elt, ElementCount  VF  )

static

Definition at line 3479 of file LoopVectorize.cpp.

References llvm::VectorType::get(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntOrPtrTy(), and llvm::ElementCount::isScalar().

Referenced by llvm::LoopVectorizationCostModel::getVectorIntrinsicCost().

processLoopInVPlanNativePath()

static bool processLoopInVPlanNativePath ( Loop *  L, PredicatedScalarEvolution &  PSE, LoopInfo *  LI, DominatorTree *  DT, LoopVectorizationLegality *  LVL, TargetTransformInfo *  TTI, TargetLibraryInfo *  TLI, DemandedBits *  DB, AssumptionCache *  AC, OptimizationRemarkEmitter *  ORE, BlockFrequencyInfo *  BFI, ProfileSummaryInfo *  PSI, LoopVectorizeHints &  Hints, LoopVectorizationRequirements &  Requirements  )

static

Definition at line 9850 of file LoopVectorize.cpp.

References assert(), llvm::LoopVectorizationCostModel::collectElementTypesForWidening(), llvm::dbgs(), llvm::VectorizationFactor::Disabled(), llvm::EnableVPlanNativePath, llvm::LoopVectorizationPlanner::executePlan(), F, llvm::PredicatedScalarEvolution::getBackedgeTakenCount(), llvm::LoopVectorizationPlanner::getBestPlanFor(), llvm::LoopVectorizationLegality::getLAI(), getScalarEpilogueLowering(), llvm::PredicatedScalarEvolution::getSE(), llvm::LoopVectorizeHints::getWidth(), LLVM_DEBUG, llvm::LoopVectorizationPlanner::planInVPlanNativePath(), llvm::LoopVectorizeHints::setAlreadyVectorized(), llvm::verifyFunction(), VPlanBuildStressTest, and llvm::VectorizationFactor::Width.

Referenced by llvm::LoopVectorizePass::processLoop().

smallestIntegerVectorType()

static Type * smallestIntegerVectorType ( Type *  T1, Type *  T2  )

static

Definition at line 3508 of file LoopVectorize.cpp.

References T1.

Referenced by llvm::InnerLoopVectorizer::truncateToMinimalBitwidths().

STATISTIC() [1/3]

STATISTIC	(	LoopsAnalyzed	,
		"Number of loops analyzed for vectorization"
	)

STATISTIC() [2/3]

STATISTIC	(	LoopsEpilogueVectorized	,
		"Number of epilogues vectorized"
	)

STATISTIC() [3/3]

STATISTIC	(	LoopsVectorized	,
		"Number of loops vectorized"
	)

useActiveLaneMask()

static bool useActiveLaneMask ( TailFoldingStyle  Style )

static

Definition at line 2155 of file LoopVectorize.cpp.

References llvm::Data, llvm::DataAndControlFlow, and llvm::DataAndControlFlowWithoutRuntimeCheck.

Referenced by llvm::VPRecipeBuilder::createBlockInMask().

useActiveLaneMaskForControlFlow()

static bool useActiveLaneMaskForControlFlow ( TailFoldingStyle  Style )

static

Definition at line 2161 of file LoopVectorize.cpp.

References llvm::DataAndControlFlow, and llvm::DataAndControlFlowWithoutRuntimeCheck.

Referenced by addCanonicalIVRecipes(), and llvm::VPRecipeBuilder::createBlockInMask().

useMaskedInterleavedAccesses()

static bool useMaskedInterleavedAccesses ( const TargetTransformInfo &  TTI )

static

Definition at line 2543 of file LoopVectorize.cpp.

References llvm::TargetTransformInfo::enableMaskedInterleavedAccessVectorization(), and EnableMaskedInterleavedMemAccesses.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationPlanner::plan(), llvm::LoopVectorizePass::processLoop(), and llvm::InnerLoopVectorizer::vectorizeInterleaveGroup().

Variable Documentation

EnableCondStoresVectorization

cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization.")) ( "enable-cond-stores-vec"  , cl::init(true)  , cl::Hidden  , cl::desc("Enable if predication of stores during vectorization.")    )

static

EnableEpilogueVectorization

cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops.")) ( "enable-epilogue-vectorization"  , cl::init(true)  , cl::Hidden  , cl::desc("Enable vectorization of epilogue loops.")    )

static

Referenced by llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

EnableIndVarRegisterHeur

cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving")) ( "enable-ind-var-reg-heur"  , cl::init(true)  , cl::Hidden  , cl::desc("Count the induction variable only once when interleaving")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

EnableInterleavedMemAccesses

cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop")) ( "enable-interleaved-mem-accesses"  , cl::init(false)  , cl::Hidden  , cl::desc("Enable vectorization on interleaved memory accesses in a loop")    )

static

Referenced by llvm::LoopVectorizePass::processLoop().

EnableLoadStoreRuntimeInterleave

cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc( "Enable runtime interleaving until load/store ports are saturated")) ( "enable-loadstore-runtime-interleave"  , cl::init(true)  , cl::Hidden  , cl::desc( "Enable runtime interleaving until load/store ports are saturated")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

EnableMaskedInterleavedMemAccesses

cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")) ( "enable-masked-interleaved-mem-accesses"  , cl::init(false)  , cl::Hidden  , cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")    )

static

An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.

Referenced by useMaskedInterleavedAccesses().

EpilogueVectorizationForceVF

cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops.")) ( "epilogue-vectorization-force-VF"  , cl::init(1)  , cl::Hidden  , cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops.")    )

static

Referenced by llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

EpilogueVectorizationMinVF

cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization.")) ( "epilogue-vectorization-minimum-VF"  , cl::init(16)  , cl::Hidden  , cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization.")    )

static

Referenced by llvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable().

ForceOrderedReductions

cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions")) ( "force-ordered-reductions"  , cl::init(false)  , cl::Hidden  , cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions")    )

static

Referenced by llvm::LoopVectorizePass::processLoop().

ForceSafeDivisor

cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc( "Override cost based safe divisor widening for div/rem instructions")) ( "force-widen-divrem-via-safe-divisor"  , cl::Hidden  , cl::desc( "Override cost based safe divisor widening for div/rem instructions")    )

static

Referenced by llvm::LoopVectorizationCostModel::isDivRemScalarWithPredication().

ForceTailFoldingStyle

cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values( clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN( TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN( TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"))) ( "force-tail-folding-style"  , cl::desc("Force the tail folding style")  , cl::init(TailFoldingStyle::None)  , cl::values( clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN( TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN( TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"))    )

static

Referenced by llvm::LoopVectorizationCostModel::getTailFoldingStyle().

ForceTargetInstructionCost

cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing.")) ( "force-target-instruction-cost"  , cl::init(0)  , cl::Hidden  , cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing.")    )

static

Referenced by llvm::LoopVectorizationCostModel::expectedCost().

ForceTargetMaxScalarInterleaveFactor

cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops.")) ( "force-target-max-scalar-interleave"  , cl::init(0)  , cl::Hidden  , cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetMaxVectorInterleaveFactor

cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops.")) ( "force-target-max-vector-interleave"  , cl::init(0)  , cl::Hidden  , cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetNumScalarRegs

cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers.")) ( "force-target-num-scalar-regs"  , cl::init(0)  , cl::Hidden  , cl::desc("A flag that overrides the target's number of scalar registers.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetNumVectorRegs

cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers.")) ( "force-target-num-vector-regs"  , cl::init(0)  , cl::Hidden  , cl::desc("A flag that overrides the target's number of vector registers.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetSupportsScalableVectors

cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc( "Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing.")) ( "force-target-supports-scalable-vectors"  , cl::init(false)  , cl::Hidden  , cl::desc( "Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing.")    )

static

InterleaveSmallLoopScalarReduction

cl::opt< bool > InterleaveSmallLoopScalarReduction("interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, cl::desc("Enable interleaving for loops with small iteration counts that " "contain scalar reductions to expose ILP.")) ( "interleave-small-loop-scalar-reduction"  , cl::init(false)  , cl::Hidden  , cl::desc("Enable interleaving for loops with small iteration counts that " "contain scalar reductions to expose ILP.")    )

static

Interleave small loops with scalar reductions.

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

LLVMLoopVectorizeFollowupAll

const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all"

Metadata attribute names

Definition at line 164 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::executePlan(), and llvm::LoopVectorizePass::processLoop().

LLVMLoopVectorizeFollowupEpilogue

const char LLVMLoopVectorizeFollowupEpilogue[]

Initial value:

=
    "llvm.loop.vectorize.followup_epilogue"

Definition at line 167 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizePass::processLoop().

LLVMLoopVectorizeFollowupVectorized

const char LLVMLoopVectorizeFollowupVectorized[]

Initial value:

=
    "llvm.loop.vectorize.followup_vectorized"

Definition at line 165 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::executePlan().

LoopVectorizeWithBlockFrequency

cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions.")) ( "loop-vectorize-with-block-frequency"  , cl::init(true)  , cl::Hidden  , cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions.")    )

static

Referenced by getSmallBestKnownTC().

MaximizeBandwidth

cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop.")) ( "vectorizer-maximize-bandwidth"  , cl::init(false)  , cl::Hidden  , cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop.")    )

static

MaxNestedScalarReductionIC

cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop.")) ( "max-nested-scalar-reduction-interleave"  , cl::init(2)  , cl::Hidden  , cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

NumberOfStoresToPredicate

cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if.")) ( "vectorize-num-stores-pred"  , cl::init(1)  , cl::Hidden  , cl::desc("Max number of stores to be predicated behind an if.")    )

static

The number of stores in a loop that are allowed to need predication.

PreferInLoopReductions

cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference.")) ( "prefer-inloop-reductions"  , cl::init(false)  , cl::Hidden  , cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference.")    )

static

Referenced by llvm::LoopVectorizationCostModel::collectElementTypesForWidening(), and llvm::LoopVectorizationCostModel::collectInLoopReductions().

PreferPredicatedReductionSelect

cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc( "Prefer predicating a reduction operation over an after loop select.")) ( "prefer-predicated-reduction-select"  , cl::init(false)  , cl::Hidden  , cl::desc( "Prefer predicating a reduction operation over an after loop select.")    )

static

Referenced by llvm::InnerLoopVectorizer::fixReduction().

PreferPredicateOverEpilogue

cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails."))) ( "prefer-predicate-over-epilogue"  , cl::init(PreferPredicateTy::ScalarEpilogue)  , cl::Hidden  , cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop.")  , cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails."))    )

static

Referenced by getScalarEpilogueLowering().

PrintVPlansInDotFormat

cl::opt< bool > PrintVPlansInDotFormat("vplan-print-in-dot-format", cl::Hidden, cl::desc("Use dot format instead of plain text when dumping VPlans")) ( "vplan-print-in-dot-format"  , cl::Hidden  , cl::desc("Use dot format instead of plain text when dumping VPlans")    )

static

Referenced by llvm::LoopVectorizationPlanner::printPlans().

SmallLoopCost

cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc( "The cost of a loop that is considered 'small' by the interleaver.")) ( "small-loop-cost"  , cl::init(20)  , cl::Hidden  , cl::desc( "The cost of a loop that is considered 'small' by the interleaver.")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

TinyTripCountInterleaveThreshold

cl::opt< unsigned > TinyTripCountInterleaveThreshold("tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden, cl::desc("We don't interleave loops with a estimated constant trip count " "below this number")) ( "tiny-trip-count-interleave-threshold"  , cl::init(128)  , cl::Hidden  , cl::desc("We don't interleave loops with a estimated constant trip count " "below this number")    )

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

TinyTripCountVectorThreshold

cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred.")) ( "vectorizer-min-trip-count"  , cl::init(16)  , cl::Hidden  , cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred.")    )

static

Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.

Referenced by llvm::LoopVectorizePass::processLoop().

VectorizeMemoryCheckThreshold

cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks")) ( "vectorize-memory-check-threshold"  , cl::init(128)  , cl::Hidden  , cl::desc("The maximum allowed number of runtime memory checks")    )

static

Referenced by areRuntimeChecksProfitable().

VerboseDebug

const char VerboseDebug[] = DEBUG_TYPE "-verbose"

Definition at line 159 of file LoopVectorize.cpp.

VPlanBuildStressTest

cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path).")) ( "vplan-build-stress-test"  , cl::init(false)  , cl::Hidden  , cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path).")    )

static

Referenced by collectSupportedLoops(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), and processLoopInVPlanNativePath().