LLVM  14.0.0git
Classes | Namespaces | Macros | Typedefs | Enumerations | Functions | Variables
LoopVectorize.cpp File Reference
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanHCFGBuilder.h"
#include "VPlanPredicator.h"
#include "VPlanTransforms.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/Verifier.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <functional>
#include <iterator>
#include <limits>
#include <memory>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes

class  llvm::InnerLoopVectorizer
 InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...
 
class  llvm::InnerLoopUnroller
 
struct  llvm::EpilogueLoopVectorizationInfo
 Encapsulate information regarding vectorization of a loop and its epilogue. More...
 
class  llvm::InnerLoopAndEpilogueVectorizer
 An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its epilogue (residual) also vectorized. More...
 
class  llvm::EpilogueVectorizerMainLoop
 A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the process of vectorizing loops and their epilogues. More...
 
class  llvm::EpilogueVectorizerEpilogueLoop
 
struct  llvm::ElementCountComparator
 ElementCountComparator creates a total ordering for ElementCount for the purposes of using it in a set structure. More...
 
class  llvm::LoopVectorizationCostModel
 LoopVectorizationCostModel - estimates the expected speedups due to vectorization. More...
 
struct  llvm::LoopVectorizationCostModel::RegisterUsage
 A struct that represents some properties of the register usage of a loop. More...
 
class  GeneratedRTChecks
 Helper struct to manage generating runtime checks for vectorization. More...
 

Namespaces

 PreferPredicateTy
 
 llvm
 ---------------------— PointerInfo ------------------------------------—
 

Macros

#define LV_NAME   "loop-vectorize"
 
#define DEBUG_TYPE   LV_NAME
 

Typedefs

using llvm::ElementCountSet = SmallSet< ElementCount, 16, ElementCountComparator >
 

Enumerations

enum  PreferPredicateTy::Option { PreferPredicateTy::ScalarEpilogue = 0, PreferPredicateTy::PredicateElseScalarEpilogue, PreferPredicateTy::PredicateOrDontVectorize }
 
enum  llvm::ScalarEpilogueLowering {
  llvm::CM_ScalarEpilogueAllowed, llvm::CM_ScalarEpilogueNotAllowedOptSize, llvm::CM_ScalarEpilogueNotAllowedLowTripLoop, llvm::CM_ScalarEpilogueNotNeededUsePredicate,
  llvm::CM_ScalarEpilogueNotAllowedUsePredicate
}
 

Functions

 STATISTIC (LoopsVectorized, "Number of loops vectorized")
 
 STATISTIC (LoopsAnalyzed, "Number of loops analyzed for vectorization")
 
 STATISTIC (LoopsEpilogueVectorized, "Number of epilogues vectorized")
 
static bool hasIrregularType (Type *Ty, const DataLayout &DL)
 A helper function that returns true if the given type is irregular. More...
 
static unsigned getReciprocalPredBlockProb ()
 A helper function that returns the reciprocal of the block probability of predicated blocks. More...
 
static ConstantgetSignedIntOrFpConstant (Type *Ty, int64_t C)
 A helper function that returns an integer or floating-point constant with value C. More...
 
static Optional< unsigned > getSmallBestKnownTC (ScalarEvolution &SE, Loop *L)
 Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known. More...
 
static InstructiongetDebugLocFromInstOrOperands (Instruction *I)
 Look for a meaningful debug location on the instruction or it's operands. More...
 
static void debugVectorizationMessage (const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
 Write a DebugMsg about vectorization to the debug output stream. More...
 
static OptimizationRemarkAnalysis createLVAnalysis (const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I)
 Create an analysis remark that explains why vectorization failed. More...
 
static ValuecreateStepForVF (IRBuilder<> &B, Constant *Step, ElementCount VF)
 Return a value for Step multiplied by VF. More...
 
Valuellvm::getRuntimeVF (IRBuilder<> &B, Type *Ty, ElementCount VF)
 Return the runtime value for VF. More...
 
void llvm::reportVectorizationFailure (const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
 Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding optimization remark RemarkName. More...
 
void llvm::reportVectorizationInfo (const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
 Reports an informative message: print Msg for debugging purposes as well as an optimization remark. More...
 
static std::string getDebugLocString (const Loop *L)
 
static bool isExplicitVecOuterLoop (Loop *OuterLp, OptimizationRemarkEmitter *ORE)
 
static void collectSupportedLoops (Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
 
static bool useMaskedInterleavedAccesses (const TargetTransformInfo &TTI)
 
static void cse (BasicBlock *BB)
 Perform cse of induction variable instructions. More...
 
static TypeMaybeVectorizeType (Type *Elt, ElementCount VF)
 
static TypesmallestIntegerVectorType (Type *T1, Type *T2)
 
static TypelargestIntegerVectorType (Type *T1, Type *T2)
 
static bool mayDivideByZero (Instruction &I)
 A helper function for checking whether an integer division-related instruction may divide by zero (in which case it must be predicated if executed conditionally in the scalar code). More...
 
static const SCEVgetAddressAccessSCEV (Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
 Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence. More...
 
static bool isStrideMul (Instruction *I, LoopVectorizationLegality *Legal)
 
Passllvm::createLoopVectorizePass ()
 
Passllvm::createLoopVectorizePass (bool InterleaveOnlyWhenForced, bool VectorizeOnlyWhenForced)
 
static unsigned determineVPlanVF (const unsigned WidestVectorRegBits, LoopVectorizationCostModel &CM)
 
static void AddRuntimeUnrollDisableMetaData (Loop *L)
 
static ScalarEpilogueLowering getScalarEpilogueLowering (Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, LoopVectorizationLegality &LVL)
 
static bool processLoopInVPlanNativePath (Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
 
static void checkMixedPrecision (Loop *L, OptimizationRemarkEmitter *ORE)
 

Variables

const char VerboseDebug [] = DEBUG_TYPE "-verbose"
 
static cl::opt< bool > EnableEpilogueVectorization ("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
 
static cl::opt< unsigned > EpilogueVectorizationForceVF ("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
 
static cl::opt< unsigned > EpilogueVectorizationMinVF ("epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
 
static cl::opt< unsigned > TinyTripCountVectorThreshold ("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
 Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred. More...
 
static cl::opt< unsigned > PragmaVectorizeMemoryCheckThreshold ("pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks with a " "vectorize(enable) pragma."))
 
static cl::opt< PreferPredicateTy::OptionPreferPredicateOverEpilogue ("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
 
static cl::opt< bool > MaximizeBandwidth ("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
 
static cl::opt< bool > EnableInterleavedMemAccesses ("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
 
static cl::opt< bool > EnableMaskedInterleavedMemAccesses ("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
 An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps. More...
 
static cl::opt< unsigned > TinyTripCountInterleaveThreshold ("tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden, cl::desc("We don't interleave loops with a estimated constant trip count " "below this number"))
 
static cl::opt< unsigned > ForceTargetNumScalarRegs ("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
 
static cl::opt< unsigned > ForceTargetNumVectorRegs ("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
 
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor ("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
 
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor ("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
 
static cl::opt< unsigned > ForceTargetInstructionCost ("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
 
static cl::opt< bool > ForceTargetSupportsScalableVectors ("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
 
static cl::opt< unsigned > SmallLoopCost ("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
 
static cl::opt< bool > LoopVectorizeWithBlockFrequency ("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
 
static cl::opt< bool > EnableLoadStoreRuntimeInterleave ("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
 
static cl::opt< bool > InterleaveSmallLoopScalarReduction ("interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, cl::desc("Enable interleaving for loops with small iteration counts that " "contain scalar reductions to expose ILP."))
 Interleave small loops with scalar reductions. More...
 
static cl::opt< unsigned > NumberOfStoresToPredicate ("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
 The number of stores in a loop that are allowed to need predication. More...
 
static cl::opt< bool > EnableIndVarRegisterHeur ("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
 
static cl::opt< bool > EnableCondStoresVectorization ("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
 
static cl::opt< unsigned > MaxNestedScalarReductionIC ("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
 
static cl::opt< bool > PreferInLoopReductions ("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
 
static cl::opt< bool > ForceOrderedReductions ("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
 
static cl::opt< bool > PreferPredicatedReductionSelect ("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
 
cl::opt< bool > EnableVPlanNativePath ("enable-vplan-native-path", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
 
cl::opt< bool > EnableVPlanPredication ("enable-vplan-predication", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path predicator with " "support for outer loop vectorization."))
 
static cl::opt< bool > VPlanBuildStressTest ("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
 
cl::opt< bool > PrintVPlansInDotFormat ("vplan-print-in-dot-format", cl::init(false), cl::Hidden, cl::desc("Use dot format instead of plain text when dumping VPlans"))
 
static const char lv_name [] = "Loop Vectorization"
 
const char LLVMLoopVectorizeFollowupAll [] = "llvm.loop.vectorize.followup_all"
 
const char LLVMLoopVectorizeFollowupVectorized []
 
const char LLVMLoopVectorizeFollowupEpilogue []
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   LV_NAME

Definition at line 159 of file LoopVectorize.cpp.

◆ LV_NAME

#define LV_NAME   "loop-vectorize"

Definition at line 158 of file LoopVectorize.cpp.

Function Documentation

◆ AddRuntimeUnrollDisableMetaData()

static void AddRuntimeUnrollDisableMetaData ( Loop L)
static

◆ checkMixedPrecision()

static void checkMixedPrecision ( Loop L,
OptimizationRemarkEmitter ORE 
)
static

◆ collectSupportedLoops()

static void collectSupportedLoops ( Loop L,
LoopInfo LI,
OptimizationRemarkEmitter ORE,
SmallVectorImpl< Loop * > &  V 
)
static

◆ createLVAnalysis()

static OptimizationRemarkAnalysis createLVAnalysis ( const char *  PassName,
StringRef  RemarkName,
Loop TheLoop,
Instruction I 
)
static

Create an analysis remark that explains why vectorization failed.

PassName is the name of the pass (e.g. can be AlwaysPrint). RemarkName is the identifier for the remark. If I is passed it is an instruction that prevents vectorization. Otherwise TheLoop is used for the location of the remark.

Returns
the remark object that can be streamed to.

Definition at line 1088 of file LoopVectorize.cpp.

References DL, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getStartLoc(), I, and PassName.

Referenced by llvm::reportVectorizationFailure(), and llvm::reportVectorizationInfo().

◆ createStepForVF()

static Value* createStepForVF ( IRBuilder<> &  B,
Constant Step,
ElementCount  VF 
)
static

◆ cse()

static void cse ( BasicBlock BB)
static

◆ debugVectorizationMessage()

static void debugVectorizationMessage ( const StringRef  Prefix,
const StringRef  DebugMsg,
Instruction I 
)
static

Write a DebugMsg about vectorization to the debug output stream.

If I is passed, the message relates to that particular instruction.

Definition at line 1069 of file LoopVectorize.cpp.

References llvm::dbgs(), I, and llvm::cl::Prefix.

Referenced by llvm::reportVectorizationFailure(), and llvm::reportVectorizationInfo().

◆ determineVPlanVF()

static unsigned determineVPlanVF ( const unsigned  WidestVectorRegBits,
LoopVectorizationCostModel CM 
)
static

◆ getAddressAccessSCEV()

static const SCEV* getAddressAccessSCEV ( Value Ptr,
LoopVectorizationLegality Legal,
PredicatedScalarEvolution PSE,
const Loop TheLoop 
)
static

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.

This SCEV can be sent to the Target in order to estimate the address calculation cost.

Definition at line 6992 of file LoopVectorize.cpp.

References llvm::PredicatedScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSE(), i, llvm::LoopVectorizationLegality::isInductionVariable(), llvm::InnerLoopVectorizer::Legal, and llvm::InnerLoopVectorizer::PSE.

◆ getDebugLocFromInstOrOperands()

static Instruction* getDebugLocFromInstOrOperands ( Instruction I)
static

◆ getDebugLocString()

static std::string getDebugLocString ( const Loop L)
static

◆ getReciprocalPredBlockProb()

static unsigned getReciprocalPredBlockProb ( )
static

A helper function that returns the reciprocal of the block probability of predicated blocks.

If we return X, we are assuming the predicated block will execute once for every X iterations of the loop header.

TODO: We should use actual block probability here, if available. Currently, we always assume predicated blocks have a 50% chance of executing.

Definition at line 394 of file LoopVectorize.cpp.

◆ getScalarEpilogueLowering()

static ScalarEpilogueLowering getScalarEpilogueLowering ( Function F,
Loop L,
LoopVectorizeHints Hints,
ProfileSummaryInfo PSI,
BlockFrequencyInfo BFI,
TargetTransformInfo TTI,
TargetLibraryInfo TLI,
AssumptionCache AC,
LoopInfo LI,
ScalarEvolution SE,
DominatorTree DT,
LoopVectorizationLegality LVL 
)
static

◆ getSignedIntOrFpConstant()

static Constant* getSignedIntOrFpConstant ( Type Ty,
int64_t  C 
)
static

A helper function that returns an integer or floating-point constant with value C.

Definition at line 398 of file LoopVectorize.cpp.

References llvm::ConstantFP::get(), llvm::ConstantInt::getSigned(), and llvm::Type::isIntegerTy().

Referenced by llvm::InnerLoopVectorizer::buildScalarSteps().

◆ getSmallBestKnownTC()

static Optional<unsigned> getSmallBestKnownTC ( ScalarEvolution SE,
Loop L 
)
static

Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known.

2) Returns expected trip count according to profile data if any. 3) Returns upper bound estimate if it is known. 4) Returns None if all of the above failed.

Definition at line 409 of file LoopVectorize.cpp.

References llvm::getLoopEstimatedTripCount(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSmallConstantTripCount(), LoopVectorizeWithBlockFrequency, and llvm::None.

Referenced by llvm::LoopVectorizePass::processLoop(), and llvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ hasIrregularType()

static bool hasIrregularType ( Type Ty,
const DataLayout DL 
)
static

A helper function that returns true if the given type is irregular.

The type is irregular if its allocated size doesn't equal the store size of an element of the corresponding vector type.

Definition at line 381 of file LoopVectorize.cpp.

References DL.

Referenced by llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), and llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened().

◆ isExplicitVecOuterLoop()

static bool isExplicitVecOuterLoop ( Loop OuterLp,
OptimizationRemarkEmitter ORE 
)
static

◆ isStrideMul()

static bool isStrideMul ( Instruction I,
LoopVectorizationLegality Legal 
)
static

◆ largestIntegerVectorType()

static Type* largestIntegerVectorType ( Type T1,
Type T2 
)
static

Definition at line 3974 of file LoopVectorize.cpp.

References T1.

◆ MaybeVectorizeType()

static Type* MaybeVectorizeType ( Type Elt,
ElementCount  VF 
)
static

◆ mayDivideByZero()

static bool mayDivideByZero ( Instruction I)
static

A helper function for checking whether an integer division-related instruction may divide by zero (in which case it must be predicated if executed conditionally in the scalar code).

TODO: It may be worthwhile to generalize and check isKnownNonZero(). Non-zero divisors that are non compile-time constants will not be converted into multiplication, so we will still end up scalarizing the division, but can do so w/o predication.

Definition at line 4852 of file LoopVectorize.cpp.

References assert(), and I.

Referenced by llvm::LoopVectorizationCostModel::isScalarWithPredication().

◆ processLoopInVPlanNativePath()

static bool processLoopInVPlanNativePath ( Loop L,
PredicatedScalarEvolution PSE,
LoopInfo LI,
DominatorTree DT,
LoopVectorizationLegality LVL,
TargetTransformInfo TTI,
TargetLibraryInfo TLI,
DemandedBits DB,
AssumptionCache AC,
OptimizationRemarkEmitter ORE,
BlockFrequencyInfo BFI,
ProfileSummaryInfo PSI,
LoopVectorizeHints Hints,
LoopVectorizationRequirements Requirements 
)
static

◆ smallestIntegerVectorType()

static Type* smallestIntegerVectorType ( Type T1,
Type T2 
)
static

Definition at line 3968 of file LoopVectorize.cpp.

References T1.

Referenced by llvm::InnerLoopVectorizer::truncateToMinimalBitwidths().

◆ STATISTIC() [1/3]

STATISTIC ( LoopsAnalyzed  ,
"Number of loops analyzed for vectorization"   
)

◆ STATISTIC() [2/3]

STATISTIC ( LoopsEpilogueVectorized  ,
"Number of epilogues vectorized"   
)

◆ STATISTIC() [3/3]

STATISTIC ( LoopsVectorized  ,
"Number of loops vectorized"   
)

◆ useMaskedInterleavedAccesses()

static bool useMaskedInterleavedAccesses ( const TargetTransformInfo TTI)
static

Variable Documentation

◆ EnableCondStoresVectorization

cl::opt<bool> EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static

◆ EnableEpilogueVectorization

cl::opt<bool> EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static

◆ EnableIndVarRegisterHeur

cl::opt<bool> EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static

◆ EnableInterleavedMemAccesses

cl::opt<bool> EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static

◆ EnableLoadStoreRuntimeInterleave

cl::opt<bool> EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc( "Enable runtime interleaving until load/store ports are saturated"))
static

◆ EnableMaskedInterleavedMemAccesses

cl::opt<bool> EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
static

An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.

Referenced by useMaskedInterleavedAccesses().

◆ EnableVPlanNativePath

cl::opt<bool> EnableVPlanNativePath("enable-vplan-native-path", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))

◆ EnableVPlanPredication

cl::opt<bool> EnableVPlanPredication("enable-vplan-predication", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path predicator with " "support for outer loop vectorization."))

◆ EpilogueVectorizationForceVF

cl::opt<unsigned> EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static

◆ EpilogueVectorizationMinVF

cl::opt<unsigned> EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static

◆ ForceOrderedReductions

cl::opt<bool> ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static

◆ ForceTargetInstructionCost

cl::opt<unsigned> ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
static

◆ ForceTargetMaxScalarInterleaveFactor

cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static

◆ ForceTargetMaxVectorInterleaveFactor

cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static

◆ ForceTargetNumScalarRegs

cl::opt<unsigned> ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static

◆ ForceTargetNumVectorRegs

cl::opt<unsigned> ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static

◆ ForceTargetSupportsScalableVectors

cl::opt<bool> ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc( "Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static

◆ InterleaveSmallLoopScalarReduction

cl::opt<bool> InterleaveSmallLoopScalarReduction("interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, cl::desc("Enable interleaving for loops with small iteration counts that " "contain scalar reductions to expose ILP."))
static

Interleave small loops with scalar reductions.

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ LLVMLoopVectorizeFollowupAll

const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all"

Metadata attribute names

Definition at line 167 of file LoopVectorize.cpp.

Referenced by llvm::InnerLoopVectorizer::completeLoopSkeleton(), and llvm::LoopVectorizePass::processLoop().

◆ LLVMLoopVectorizeFollowupEpilogue

const char LLVMLoopVectorizeFollowupEpilogue[]
Initial value:
=
"llvm.loop.vectorize.followup_epilogue"

Definition at line 170 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizePass::processLoop().

◆ LLVMLoopVectorizeFollowupVectorized

const char LLVMLoopVectorizeFollowupVectorized[]
Initial value:
=
"llvm.loop.vectorize.followup_vectorized"

Definition at line 168 of file LoopVectorize.cpp.

Referenced by llvm::InnerLoopVectorizer::completeLoopSkeleton().

◆ LoopVectorizeWithBlockFrequency

cl::opt<bool> LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static

Referenced by getSmallBestKnownTC().

◆ lv_name

const char lv_name[] = "Loop Vectorization"
static

Definition at line 7939 of file LoopVectorize.cpp.

◆ MaximizeBandwidth

cl::opt<bool> MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static

◆ MaxNestedScalarReductionIC

cl::opt<unsigned> MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static

◆ NumberOfStoresToPredicate

cl::opt<unsigned> NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
static

The number of stores in a loop that are allowed to need predication.

◆ PragmaVectorizeMemoryCheckThreshold

cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold("pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks with a " "vectorize(enable) pragma."))
static

◆ PreferInLoopReductions

cl::opt<bool> PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static

◆ PreferPredicatedReductionSelect

cl::opt<bool> PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc( "Prefer predicating a reduction operation over an after loop select."))
static

◆ PreferPredicateOverEpilogue

cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static

◆ PrintVPlansInDotFormat

cl::opt<bool> PrintVPlansInDotFormat("vplan-print-in-dot-format", cl::init(false), cl::Hidden, cl::desc("Use dot format instead of plain text when dumping VPlans"))

◆ SmallLoopCost

cl::opt<unsigned> SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc( "The cost of a loop that is considered 'small' by the interleaver."))
static

◆ TinyTripCountInterleaveThreshold

cl::opt<unsigned> TinyTripCountInterleaveThreshold("tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden, cl::desc("We don't interleave loops with a estimated constant trip count " "below this number"))
static

◆ TinyTripCountVectorThreshold

cl::opt<unsigned> TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
static

Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.

Referenced by llvm::LoopVectorizePass::processLoop().

◆ VerboseDebug

const char VerboseDebug[] = DEBUG_TYPE "-verbose"

◆ VPlanBuildStressTest

cl::opt<bool> VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static