LLVM
13.0.0git
|
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...
Public Types | |
using | VectorParts = SmallVector< Value *, 2 > |
A type for vectorized values in the new loop. More... | |
Public Member Functions | |
InnerLoopVectorizer (Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationLegality *LVL, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks) | |
virtual | ~InnerLoopVectorizer ()=default |
virtual BasicBlock * | createVectorizedLoopSkeleton () |
Create a new empty loop that will contain vectorized instructions later on, while the old loop will be used as the scalar remainder. More... | |
void | widenInstruction (Instruction &I, VPValue *Def, VPUser &Operands, VPTransformState &State) |
Widen a single instruction within the innermost loop. More... | |
void | widenCallInstruction (CallInst &I, VPValue *Def, VPUser &ArgOperands, VPTransformState &State) |
Widen a single call instruction within the innermost loop. More... | |
void | widenSelectInstruction (SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State) |
Widen a single select instruction within the innermost loop. More... | |
void | fixVectorizedLoop (VPTransformState &State) |
Fix the vectorized code, taking care of header phi's, live-outs, and more. More... | |
bool | areSafetyChecksAdded () |
void | widenGEP (GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices, unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State) |
Vectorize a single GetElementPtrInst based on information gathered and decisions taken during planning. More... | |
void | widenPHIInstruction (Instruction *PN, RecurrenceDescriptor *RdxDesc, VPWidenPHIRecipe *PhiR, VPTransformState &State) |
Vectorize a single PHINode in a block. More... | |
void | scalarizeInstruction (Instruction *Instr, VPValue *Def, VPUser &Operands, const VPIteration &Instance, bool IfPredicateInstr, VPTransformState &State) |
A helper function to scalarize a single Instruction in the innermost loop. More... | |
void | widenIntOrFpInduction (PHINode *IV, Value *Start, TruncInst *Trunc, VPValue *Def, VPValue *CastDef, VPTransformState &State) |
Widen an integer or floating-point induction variable IV . More... | |
void | packScalarIntoVectorValue (VPValue *Def, const VPIteration &Instance, VPTransformState &State) |
Construct the vector value of a scalarized value V one lane at a time. More... | |
void | vectorizeInterleaveGroup (const InterleaveGroup< Instruction > *Group, ArrayRef< VPValue * > VPDefs, VPTransformState &State, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *BlockInMask=nullptr) |
Try to vectorize interleaved access group Group with the base address given in Addr , optionally masking the vector operations if BlockInMask is non-null. More... | |
void | vectorizeMemoryInstruction (Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr, VPValue *StoredValue, VPValue *BlockInMask) |
Vectorize Load and Store instructions with the base address given in Addr , optionally masking the vector operations if BlockInMask is non-null. More... | |
void | setDebugLocFromInst (IRBuilder<> &B, const Value *Ptr) |
Set the debug location in the builder using the debug location in the instruction. More... | |
void | fixNonInductionPHIs (VPTransformState &State) |
Fix the non-induction PHIs in the OrigPHIsToFix vector. More... | |
virtual Value * | getBroadcastInstrs (Value *V) |
Create a broadcast instruction. More... | |
Protected Types | |
using | PhiVector = SmallVector< PHINode *, 4 > |
A small list of PHINodes. More... | |
using | ScalarParts = SmallVector< SmallVector< Value *, 4 >, 2 > |
A type for scalarized values in the new loop. More... | |
Protected Member Functions | |
void | fixupIVUsers (PHINode *OrigPhi, const InductionDescriptor &II, Value *CountRoundDown, Value *EndValue, BasicBlock *MiddleBlock) |
Set up the values of the IVs correctly when exiting the vector loop. More... | |
PHINode * | createInductionVariable (Loop *L, Value *Start, Value *End, Value *Step, Instruction *DL) |
Create a new induction variable inside L. More... | |
void | fixCrossIterationPHIs (VPTransformState &State) |
Handle all cross-iteration phis in the header. More... | |
void | fixFirstOrderRecurrence (PHINode *Phi, VPTransformState &State) |
Fix a first-order recurrence. More... | |
void | fixReduction (PHINode *Phi, VPTransformState &State) |
Fix a reduction cross-iteration phi. More... | |
void | clearReductionWrapFlags (RecurrenceDescriptor &RdxDesc, VPTransformState &State) |
Clear NSW/NUW flags from reduction instructions if necessary. More... | |
void | fixLCSSAPHIs (VPTransformState &State) |
Fixup the LCSSA phi nodes in the unique exit block. More... | |
void | sinkScalarOperands (Instruction *PredInst) |
Iteratively sink the scalarized operands of a predicated instruction into the block that was created for it. More... | |
void | truncateToMinimalBitwidths (VPTransformState &State) |
Shrinks vector element sizes to the smallest bitwidth they can be legally represented as. More... | |
virtual Value * | getStepVector (Value *Val, int StartIdx, Value *Step, Instruction::BinaryOps Opcode=Instruction::BinaryOpsEnd) |
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...) to each vector element of Val. More... | |
void | buildScalarSteps (Value *ScalarIV, Value *Step, Instruction *EntryVal, const InductionDescriptor &ID, VPValue *Def, VPValue *CastDef, VPTransformState &State) |
Compute scalar induction steps. More... | |
void | createVectorIntOrFpInductionPHI (const InductionDescriptor &II, Value *Step, Value *Start, Instruction *EntryVal, VPValue *Def, VPValue *CastDef, VPTransformState &State) |
Create a vector induction phi node based on an existing scalar one. More... | |
bool | shouldScalarizeInstruction (Instruction *I) const |
Returns true if an instruction I should be scalarized instead of vectorized for the chosen vectorization factor. More... | |
bool | needsScalarInduction (Instruction *IV) const |
Returns true if we should generate a scalar version of IV . More... | |
void | recordVectorLoopValueForInductionCast (const InductionDescriptor &ID, const Instruction *EntryVal, Value *VectorLoopValue, VPValue *CastDef, VPTransformState &State, unsigned Part, unsigned Lane=UINT_MAX) |
If there is a cast involved in the induction variable ID , which should be ignored in the vectorized loop body, this function records the VectorLoopValue of the respective Phi also as the VectorLoopValue of the cast. More... | |
virtual Value * | reverseVector (Value *Vec) |
Generate a shuffle sequence that will reverse the vector Vec. More... | |
Value * | getOrCreateTripCount (Loop *NewLoop) |
Returns (and creates if needed) the original loop trip count. More... | |
Value * | getOrCreateVectorTripCount (Loop *NewLoop) |
Returns (and creates if needed) the trip count of the widened loop. More... | |
Value * | createBitOrPointerCast (Value *V, VectorType *DstVTy, const DataLayout &DL) |
Returns a bitcasted value to the requested vector type. More... | |
void | emitMinimumIterationCountCheck (Loop *L, BasicBlock *Bypass) |
Emit a bypass check to see if the vector trip count is zero, including if it overflows. More... | |
BasicBlock * | emitSCEVChecks (Loop *L, BasicBlock *Bypass) |
Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct. More... | |
BasicBlock * | emitMemRuntimeChecks (Loop *L, BasicBlock *Bypass) |
Emit bypass checks to check any memory assumptions we may have made. More... | |
Value * | emitTransformedIndex (IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL, const InductionDescriptor &ID) const |
Compute the transformed value of Index at offset StartValue using step StepValue. More... | |
Loop * | createVectorLoopSkeleton (StringRef Prefix) |
Emit basic blocks (prefixed with Prefix ) for the iteration check, vector loop preheader, middle block and scalar preheader. More... | |
void | createInductionResumeValues (Loop *L, Value *VectorTripCount, std::pair< BasicBlock *, Value * > AdditionalBypass={nullptr, nullptr}) |
Create new phi nodes for the induction variables to resume iteration count in the scalar epilogue, from where the vectorized loop left off (given by VectorTripCount ). More... | |
BasicBlock * | completeLoopSkeleton (Loop *L, MDNode *OrigLoopID) |
Complete the loop skeleton by adding debug MDs, creating appropriate conditional branches in the middle block, preparing the builder and running the verifier. More... | |
void | addNewMetadata (Instruction *To, const Instruction *Orig) |
Add additional metadata to To that was not present on Orig . More... | |
void | addMetadata (Instruction *To, Instruction *From) |
Add metadata from one instruction to another. More... | |
void | addMetadata (ArrayRef< Value * > To, Instruction *From) |
Similar to the previous function but it adds the metadata to a vector of instructions. More... | |
virtual void | printDebugTracesAtStart () |
Allow subclasses to override and print debug traces before/after vplan execution, when trace information is requested. More... | |
virtual void | printDebugTracesAtEnd () |
Protected Attributes | |
Loop * | OrigLoop |
The original loop. More... | |
PredicatedScalarEvolution & | PSE |
A wrapper around ScalarEvolution used to add runtime SCEV checks. More... | |
LoopInfo * | LI |
Loop Info. More... | |
DominatorTree * | DT |
Dominator Tree. More... | |
AAResults * | AA |
Alias Analysis. More... | |
const TargetLibraryInfo * | TLI |
Target Library Info. More... | |
const TargetTransformInfo * | TTI |
Target Transform Info. More... | |
AssumptionCache * | AC |
Assumption Cache. More... | |
OptimizationRemarkEmitter * | ORE |
Interface to emit optimization remarks. More... | |
std::unique_ptr< LoopVersioning > | LVer |
LoopVersioning. More... | |
ElementCount | VF |
The vectorization SIMD factor to use. More... | |
unsigned | UF |
The vectorization unroll factor to use. More... | |
IRBuilder | Builder |
The builder that we use. More... | |
BasicBlock * | LoopVectorPreHeader |
The vector-loop preheader. More... | |
BasicBlock * | LoopScalarPreHeader |
The scalar-loop preheader. More... | |
BasicBlock * | LoopMiddleBlock |
Middle Block between the vector and the scalar. More... | |
BasicBlock * | LoopExitBlock |
The (unique) ExitBlock of the scalar loop. More... | |
BasicBlock * | LoopVectorBody |
The vector loop body. More... | |
BasicBlock * | LoopScalarBody |
The scalar loop body. More... | |
SmallVector< BasicBlock *, 4 > | LoopBypassBlocks |
A list of all bypass blocks. The first block is the entry of the loop. More... | |
PHINode * | Induction = nullptr |
The new Induction variable which was added to the new block. More... | |
PHINode * | OldInduction = nullptr |
The induction variable of the old basic block. More... | |
SmallVector< Instruction *, 4 > | PredicatedInstructions |
Store instructions that were predicated. More... | |
Value * | TripCount = nullptr |
Trip count of the original loop. More... | |
Value * | VectorTripCount = nullptr |
Trip count of the widened loop (TripCount - TripCount % (VF*UF)) More... | |
LoopVectorizationLegality * | Legal |
The legality analysis. More... | |
LoopVectorizationCostModel * | Cost |
The profitablity analysis. More... | |
bool | AddedSafetyChecks = false |
DenseMap< PHINode *, Value * > | IVEndValues |
SmallVector< PHINode *, 8 > | OrigPHIsToFix |
BlockFrequencyInfo * | BFI |
BFI and PSI are used to check for profile guided size optimizations. More... | |
ProfileSummaryInfo * | PSI |
bool | OptForSizeBasedOnProfile |
GeneratedRTChecks & | RTChecks |
Structure to hold information about generated runtime checks, responsible for cleaning the checks, if vectorization turns out unprofitable. More... | |
Friends | |
class | LoopVectorizationPlanner |
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF).
This class performs the widening of scalars into vectors, or multiple scalars. This class also implements the following features:
Definition at line 454 of file LoopVectorize.cpp.
|
protected |
A small list of PHINodes.
Definition at line 576 of file LoopVectorize.cpp.
|
protected |
A type for scalarized values in the new loop.
Each value from the original loop, when scalarized, is represented by UF x VF scalar values in the new unrolled loop, where UF is the unroll factor and VF is the vectorization factor.
Definition at line 582 of file LoopVectorize.cpp.
using llvm::InnerLoopVectorizer::VectorParts = SmallVector<Value *, 2> |
A type for vectorized values in the new loop.
Each value from the original loop, when vectorized, is represented by UF vector values in the new unrolled loop, where UF is the unroll factor.
Definition at line 506 of file LoopVectorize.cpp.
|
inline |
Definition at line 456 of file LoopVectorize.cpp.
References BFI, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::IRPass, OptForSizeBasedOnProfile, OrigLoop, PSI, and llvm::shouldOptimizeForSize().
|
virtualdefault |
|
protected |
Similar to the previous function but it adds the metadata to a vector of instructions.
Definition at line 1167 of file LoopVectorize.cpp.
References addMetadata(), From, and I.
|
protected |
Add metadata from one instruction to another.
This includes both the original MDs from From
and additional ones (
Definition at line 1161 of file LoopVectorize.cpp.
References addNewMetadata(), From, and llvm::propagateMetadata().
Referenced by addMetadata(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), and widenSelectInstruction().
|
protected |
Add additional metadata to To
that was not present on Orig
.
Currently this is used to add the noalias annotations based on the inserted memchecks. Use this for instructions that are cloned into the vector loop.
Definition at line 1153 of file LoopVectorize.cpp.
References LVer.
Referenced by addMetadata(), and scalarizeInstruction().
|
inline |
Definition at line 501 of file LoopVectorize.cpp.
References AddedSafetyChecks.
Referenced by llvm::LoopVectorizePass::processLoop().
|
protected |
Compute scalar induction steps.
ScalarIV
is the scalar induction variable on which to base the steps, Step
is the size of the step, and EntryVal
is the value from the original loop that maps to the steps. Note that EntryVal
doesn't have to be an induction variable - it can also be a truncate instruction.
Definition at line 2520 of file LoopVectorize.cpp.
References llvm::MCID::Add, assert(), Builder, Cost, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateSIToFP(), createStepForVF(), llvm::IRBuilderBase::CreateStepVector(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::tgtok::Def, llvm::IntegerType::get(), llvm::ConstantInt::get(), llvm::VectorType::get(), llvm::Type::getContext(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::Type::getScalarSizeInBits(), llvm::Type::getScalarType(), getSignedIntOrFpConstant(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::ElementCount::isVector(), llvm::Mul, recordVectorLoopValueForInductionCast(), llvm::VPTransformState::set(), UF, and VF.
|
protected |
Clear NSW/NUW flags from reduction instructions if necessary.
Definition at line 4451 of file LoopVectorize.cpp.
References llvm::Add, assert(), llvm::LoopBase< BlockT, LoopT >::contains(), llvm::VPTransformState::get(), llvm::RecurrenceDescriptor::getLoopExitInstr(), llvm::Instruction::getParent(), llvm::RecurrenceDescriptor::getRecurrenceKind(), llvm::VPlan::getVPValue(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::Mul, OrigLoop, llvm::VPTransformState::Plan, llvm::SmallVectorImpl< T >::pop_back_val(), UF, and llvm::Value::users().
Referenced by fixReduction().
|
protected |
Complete the loop skeleton by adding debug MDs, creating appropriate conditional branches in the middle block, preparing the builder and running the verifier.
Take in the vector loop L
as argument, and return the preheader of the completed vector loop.
Definition at line 3514 of file LoopVectorize.cpp.
References assert(), Builder, Cost, llvm::CmpInst::Create(), DT, llvm::CallingConv::Fast, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::BasicBlock::getFirstInsertionPt(), llvm::Loop::getLoopID(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), getOrCreateTripCount(), getOrCreateVectorTripCount(), llvm::BasicBlock::getTerminator(), llvm::Optional< T >::getValue(), llvm::Optional< T >::hasValue(), llvm::CmpInst::ICMP_EQ, LI, LLVMLoopVectorizeFollowupAll, LLVMLoopVectorizeFollowupVectorized, LoopMiddleBlock, LoopVectorBody, LoopVectorPreHeader, llvm::makeFollowupLoopID(), ORE, OrigLoop, llvm::LoopVectorizeHints::setAlreadyVectorized(), llvm::Instruction::setDebugLoc(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Loop::setLoopID(), VectorTripCount, llvm::DominatorTreeBase< NodeT, IsPostDom >::verify(), and llvm::LoopInfoBase< BlockT, LoopT >::verify().
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Returns a bitcasted value to the requested vector type.
Also handles bitcasts of vector<float> <-> vector<pointer> types.
Definition at line 3164 of file LoopVectorize.cpp.
References assert(), Builder, llvm::IRBuilderBase::CreateBitOrPointerCast(), DL, llvm::FixedVectorType::get(), llvm::Value::getContext(), llvm::Type::getIntNTy(), llvm::Value::getType(), llvm::CastInst::isBitOrNoopPointerCastable(), llvm::Type::isFloatingPointTy(), llvm::Type::isPointerTy(), and VF.
Referenced by vectorizeInterleaveGroup().
|
protected |
Create new phi nodes for the induction variables to resume iteration count in the scalar epilogue, from where the vectorized loop left off (given by VectorTripCount
).
In cases where the loop skeleton is more complicated (eg. epilogue vectorization) and the resume values can come from an additional bypass block, the AdditionalBypass
pair provides information about the bypass block and the end value on the edge from bypass to this loop.
Definition at line 3440 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), assert(), B, BB, llvm::PHINode::Create(), DL, emitTransformedIndex(), llvm::CastInst::getCastOpcode(), llvm::Module::getDataLayout(), llvm::Instruction::getDebugLoc(), llvm::Instruction::getFastMathFlags(), llvm::InductionDescriptor::getInductionBinOp(), llvm::LoopVectorizationLegality::getInductionVars(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BasicBlock::getModule(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::getStartValue(), llvm::InductionDescriptor::getStep(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), IVEndValues, Legal, LoopBypassBlocks, LoopMiddleBlock, LoopScalarBody, LoopScalarPreHeader, OldInduction, PSE, llvm::Instruction::setDebugLoc(), llvm::PHINode::setIncomingValueForBlock(), llvm::Value::setName(), and VectorTripCount.
Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Create a new induction variable inside L.
Definition at line 3031 of file LoopVectorize.cpp.
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Create a vector induction phi node based on an existing scalar one.
EntryVal
is the value from the original loop that maps to the vector phi node, and Step
is the loop-invariant step. If EntryVal
is a truncate instruction, instead of widening the original IV, we widen a version of the IV truncated to EntryVal's
type.
Definition at line 2226 of file LoopVectorize.cpp.
|
virtual |
Create a new empty loop that will contain vectorized instructions later on, while the old loop will be used as the scalar remainder.
Control flow is generated around the vectorized (and scalar epilogue) loops consisting of various checks and bypasses. Return the pre-header block of the new loop. In the case of epilogue vectorization, this function is overriden to handle the more complex control flow around the loops.
Reimplemented in llvm::InnerLoopAndEpilogueVectorizer.
Definition at line 3573 of file LoopVectorize.cpp.
References Builder, completeLoopSkeleton(), createInductionResumeValues(), createInductionVariable(), createStepForVF(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), emitSCEVChecks(), llvm::ConstantInt::get(), getDebugLocFromInstOrOperands(), llvm::BasicBlock::getFirstInsertionPt(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getLoopID(), getOrCreateVectorTripCount(), llvm::LoopVectorizationLegality::getPrimaryInduction(), llvm::LoopVectorizationLegality::getWidestInductionType(), Induction, Legal, LoopScalarPreHeader, OldInduction, OrigLoop, llvm::IRBuilderBase::SetInsertPoint(), UF, and VF.
Referenced by llvm::LoopVectorizationPlanner::executePlan().
Emit basic blocks (prefixed with Prefix
) for the iteration check, vector loop preheader, middle block and scalar preheader.
Also allocate a loop object for the new vector loop and return it.
Definition at line 3392 of file LoopVectorize.cpp.
References llvm::LoopBase< BlockT, LoopT >::addBasicBlockToLoop(), llvm::LoopBase< BlockT, LoopT >::addChildLoop(), llvm::LoopInfoBase< BlockT, LoopT >::addTopLevelLoop(), llvm::LoopInfoBase< BlockT, LoopT >::AllocateLoop(), assert(), Builder, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), llvm::BranchInst::Create(), DT, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::LoopBase< BlockT, LoopT >::getParentLoop(), llvm::BasicBlock::getTerminator(), llvm::IRBuilderBase::getTrue(), llvm::LoopBase< BlockT, LoopT >::getUniqueExitBlock(), LI, LoopExitBlock, LoopMiddleBlock, LoopScalarBody, LoopScalarPreHeader, LoopVectorBody, LoopVectorPreHeader, OrigLoop, llvm::cl::Prefix, llvm::ReplaceInstWithInst(), llvm::Instruction::setDebugLoc(), and llvm::SplitBlock().
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Emit bypass checks to check any memory assumptions we may have made.
Returns the block containing the checks or nullptr if no checks have been added.
Definition at line 3261 of file LoopVectorize.cpp.
References AddedSafetyChecks, assert(), Cost, DEBUG_TYPE, DT, llvm::OptimizationRemarkEmitter::emit(), GeneratedRTChecks::emitMemRuntimeChecks(), EnableVPlanNativePath, llvm::LoopVectorizeHints::FK_Enabled, llvm::RuntimePointerChecking::getChecks(), llvm::LoopVectorizeHints::getForce(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::getLAI(), llvm::BasicBlock::getParent(), llvm::LoopAccessInfo::getRuntimePointerChecking(), llvm::PredicatedScalarEvolution::getSE(), llvm::Loop::getStartLoc(), llvm::Function::hasOptSize(), llvm::LoopVectorizationCostModel::Hints, Legal, LI, LoopBypassBlocks, LoopVectorPreHeader, LVer, OptForSizeBasedOnProfile, ORE, OrigLoop, PSE, and RTChecks.
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Emit a bypass check to see if the vector trip count is zero, including if it overflows.
Definition at line 3195 of file LoopVectorize.cpp.
References assert(), Builder, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), Cost, llvm::BranchInst::Create(), llvm::IRBuilderBase::CreateICmp(), createStepForVF(), DT, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::ConstantInt::get(), llvm::IRBuilderBase::getFalse(), llvm::DomTreeNodeBase< NodeT >::getIDom(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), getOrCreateTripCount(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, LI, LoopBypassBlocks, LoopExitBlock, LoopVectorPreHeader, P, llvm::DominatorTreeBase< NodeT, IsPostDom >::properlyDominates(), llvm::ReplaceInstWithInst(), llvm::LoopVectorizationCostModel::requiresScalarEpilogue(), llvm::SplitBlock(), UF, and VF.
|
protected |
Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct.
Returns the block containing the checks or nullptr if no checks have been added.
Definition at line 3237 of file LoopVectorize.cpp.
References AddedSafetyChecks, assert(), llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), Cost, DT, GeneratedRTChecks::emitSCEVChecks(), llvm::LoopVectorizeHints::FK_Enabled, llvm::LoopVectorizeHints::getForce(), llvm::BasicBlock::getParent(), llvm::Function::hasOptSize(), llvm::LoopVectorizationCostModel::Hints, LoopBypassBlocks, LoopExitBlock, LoopVectorPreHeader, OptForSizeBasedOnProfile, and RTChecks.
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protected |
Compute the transformed value of Index at offset StartValue using step StepValue.
For integer induction, returns StartValue + Index * StepValue. For pointer induction, returns StartValue[Index * StepValue]. FIXME: The newly created binary instructions should contain nsw/nuw flags, which can be found from the original scalar operations.
Definition at line 3304 of file LoopVectorize.cpp.
References assert(), B, CreateAdd(), CreateMul(), DL, llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::BasicBlock::getTerminator(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, Index, LI, llvm_unreachable, LoopVectorBody, Offset, X, and Y.
Referenced by createInductionResumeValues(), fixupIVUsers(), and widenPHIInstruction().
|
protected |
Handle all cross-iteration phis in the header.
Definition at line 4061 of file LoopVectorize.cpp.
References fixFirstOrderRecurrence(), fixReduction(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::isFirstOrderRecurrence(), llvm::LoopVectorizationLegality::isReductionVariable(), Legal, OrigLoop, and llvm::BasicBlock::phis().
Referenced by fixVectorizedLoop().
|
protected |
Fix a first-order recurrence.
This is the second phase of vectorizing this phi node.
Definition at line 4077 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), assert(), llvm::BasicBlock::begin(), Builder, llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::PoisonValue::get(), llvm::BasicBlock::getFirstInsertionPt(), llvm::PHINode::getIncomingValueForBlock(), llvm::IRBuilderBase::getInt32(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), llvm::VPlan::getVPValue(), I, llvm::Loop::isLoopInvariant(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), LI, LoopMiddleBlock, LoopScalarPreHeader, LoopVectorBody, LoopVectorPreHeader, OrigLoop, llvm::VPTransformState::Plan, llvm::Value::replaceAllUsesWith(), llvm::VPTransformState::reset(), llvm::IRBuilderBase::SetInsertPoint(), UF, and VF.
Referenced by fixCrossIterationPHIs().
|
protected |
Fixup the LCSSA phi nodes in the unique exit block.
This simply means we need to add the appropriate incoming value from the middle block as exiting edges from the scalar epilogue loop (if present) are already in place, and we exit the vector loop exclusively to the middle block.
Definition at line 4481 of file LoopVectorize.cpp.
References Builder, Cost, llvm::VPTransformState::get(), llvm::VPLane::getFirstLane(), llvm::VPLane::getLastLaneForVF(), llvm::BasicBlock::getTerminator(), llvm::VPlan::getVPValue(), llvm::Loop::isLoopInvariant(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), LoopExitBlock, LoopMiddleBlock, OrigLoop, llvm::BasicBlock::phis(), llvm::VPTransformState::Plan, llvm::IRBuilderBase::SetInsertPoint(), UF, and VF.
void InnerLoopVectorizer::fixNonInductionPHIs | ( | VPTransformState & | State | ) |
Fix the non-induction PHIs in the OrigPHIsToFix vector.
Definition at line 4573 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), Builder, llvm::VPTransformState::CFG, llvm::VPTransformState::get(), llvm::VPWidenPHIRecipe::getIncomingBlock(), llvm::VPWidenPHIRecipe::getIncomingValue(), llvm::VPUser::getNumOperands(), llvm::VPlan::getVPValue(), i, OrigPHIsToFix, llvm::VPTransformState::Plan, llvm::IRBuilderBase::SetInsertPoint(), and llvm::VPTransformState::CFGState::VPBB2IRBB.
Referenced by fixVectorizedLoop().
|
protected |
Fix a reduction cross-iteration phi.
This is the second phase of vectorizing this phi node.
Definition at line 4268 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), llvm::any_of(), assert(), Builder, clearReductionWrapFlags(), Cost, llvm::PHINode::Create(), llvm::IRBuilderBase::CreateBinOp(), llvm::createMinMaxOp(), llvm::IRBuilderBase::CreateSExt(), llvm::createTargetReduction(), llvm::IRBuilderBase::CreateTrunc(), llvm::IRBuilderBase::CreateZExt(), E, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::PHINode::getBasicBlockIndex(), llvm::RecurrenceDescriptor::getFastMathFlags(), llvm::BasicBlock::getFirstInsertionPt(), llvm::PHINode::getIncomingValueForBlock(), llvm::RecurrenceDescriptor::getLoopExitInstr(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::RecurrenceDescriptor::getOpcode(), llvm::RecurrenceDescriptor::getRecurrenceKind(), llvm::RecurrenceDescriptor::getRecurrenceStartValue(), llvm::RecurrenceDescriptor::getRecurrenceType(), llvm::LoopVectorizationLegality::getReductionVars(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::VPlan::getVPValue(), I, llvm::LoopVectorizationCostModel::isInLoopReduction(), llvm::LoopVectorizationLegality::isReductionVariable(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::RecurrenceDescriptor::isSigned(), llvm::ElementCount::isVector(), Legal, LI, LoopBypassBlocks, LoopExitBlock, LoopMiddleBlock, LoopScalarPreHeader, LoopVectorBody, OrigLoop, llvm::BasicBlock::phis(), llvm::VPTransformState::Plan, PreferPredicatedReductionSelect, llvm::TargetTransformInfo::preferPredicatedReductionSelect(), replaceUsesOfWith(), llvm::VPTransformState::reset(), setDebugLocFromInst(), llvm::IRBuilderBase::setFastMathFlags(), llvm::PHINode::setIncomingValue(), llvm::IRBuilderBase::SetInsertPoint(), UF, useOrderedReductions(), llvm::Value::users(), llvm::VPTransformState::VF, and VF.
Referenced by fixCrossIterationPHIs().
|
protected |
Set up the values of the IVs correctly when exiting the vector loop.
Definition at line 3662 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), assert(), B, llvm::LoopBase< BlockT, LoopT >::contains(), DL, emitTransformedIndex(), llvm::ConstantInt::get(), llvm::PHINode::getBasicBlockIndex(), llvm::Module::getDataLayout(), llvm::Instruction::getFastMathFlags(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::PHINode::getIncomingValueForBlock(), llvm::InductionDescriptor::getInductionBinOp(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::BasicBlock::getModule(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::getStep(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::LoopBase< BlockT, LoopT >::getUniqueExitBlock(), I, llvm::Type::isIntegerTy(), OrigLoop, PostInc, PSE, llvm::Value::setName(), and llvm::Value::users().
void InnerLoopVectorizer::fixVectorizedLoop | ( | VPTransformState & | State | ) |
Fix the vectorized code, taking care of header phi's, live-outs, and more.
Definition at line 4008 of file LoopVectorize.cpp.
References assert(), EnableVPlanNativePath, fixCrossIterationPHIs(), fixNonInductionPHIs(), llvm::ScalarEvolution::forgetLoop(), llvm::PredicatedScalarEvolution::getSE(), llvm::ElementCount::isVector(), OrigLoop, OrigPHIsToFix, PSE, truncateToMinimalBitwidths(), and VF.
Referenced by llvm::LoopVectorizationPlanner::executePlan().
Create a broadcast instruction.
This method generates a broadcast instruction (shuffle) for loop invariant values and for the induction value. If this is the induction variable then we extend it to N, N+1, ... this is needed because each iteration in the loop corresponds to a SIMD element.
Definition at line 2207 of file LoopVectorize.cpp.
References Builder, llvm::IRBuilderBase::CreateVectorSplat(), llvm::DominatorTree::dominates(), DT, llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), llvm::Loop::isLoopInvariant(), LoopVectorPreHeader, OrigLoop, llvm::IRBuilderBase::SetInsertPoint(), and VF.
Returns (and creates if needed) the original loop trip count.
Definition at line 3063 of file LoopVectorize.cpp.
References assert(), Builder, llvm::CastInst::CreatePointerCast(), DL, llvm::ScalarEvolution::getAddExpr(), llvm::PredicatedScalarEvolution::getBackedgeTakenCount(), llvm::Module::getDataLayout(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BasicBlock::getModule(), llvm::ScalarEvolution::getNoopOrZeroExtend(), llvm::ScalarEvolution::getOne(), llvm::Type::getPrimitiveSizeInBits(), llvm::PredicatedScalarEvolution::getSE(), llvm::BasicBlock::getTerminator(), llvm::ScalarEvolution::getTruncateOrNoop(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::ScalarEvolution::getTypeSizeInBits(), llvm::LoopVectorizationLegality::getWidestInductionType(), llvm::Type::isPointerTy(), Legal, PSE, and TripCount.
Referenced by completeLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::LoopVectorizationPlanner::executePlan(), and getOrCreateVectorTripCount().
Returns (and creates if needed) the trip count of the widened loop.
Definition at line 3110 of file LoopVectorize.cpp.
References assert(), Builder, Cost, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateSelect(), createStepForVF(), llvm::IRBuilderBase::CreateSub(), llvm::IRBuilderBase::CreateURem(), llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::ConstantInt::get(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), getOrCreateTripCount(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::isPowerOf2_32(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), llvm::LoopVectorizationCostModel::requiresScalarEpilogue(), UF, VectorTripCount, and VF.
Referenced by completeLoopSkeleton(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().
|
protectedvirtual |
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...) to each vector element of Val.
The sequence starts at StartIndex. Opcode
is relevant for FP induction variable.
Definition at line 2474 of file LoopVectorize.cpp.
References assert(), Builder, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateFMul(), llvm::IRBuilderBase::CreateMul(), llvm::IRBuilderBase::CreateStepVector(), llvm::IRBuilderBase::CreateUIToFP(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::IntegerType::get(), llvm::ConstantInt::get(), llvm::VectorType::get(), llvm::Type::getContext(), llvm::Type::getScalarSizeInBits(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), and llvm::Type::isIntegerTy().
|
protected |
Returns true if we should generate a scalar version of IV
.
Definition at line 2316 of file LoopVectorize.cpp.
References llvm::any_of(), llvm::LoopBase< BlockT, LoopT >::contains(), I, OrigLoop, shouldScalarizeInstruction(), and llvm::Value::users().
void InnerLoopVectorizer::packScalarIntoVectorValue | ( | VPValue * | Def, |
const VPIteration & | Instance, | ||
VPTransformState & | State | ||
) |
Construct the vector value of a scalarized value V
one lane at a time.
Definition at line 2601 of file LoopVectorize.cpp.
References llvm::VPTransformState::Builder, Builder, llvm::IRBuilderBase::CreateInsertElement(), llvm::tgtok::Def, llvm::VPTransformState::get(), llvm::VPLane::getAsRuntimeExpr(), llvm::VPIteration::Lane, llvm::VPIteration::Part, llvm::VPTransformState::set(), and VF.
Referenced by llvm::VPReplicateRecipe::execute().
|
inlineprotectedvirtual |
Reimplemented in llvm::EpilogueVectorizerEpilogueLoop, and llvm::EpilogueVectorizerMainLoop.
Definition at line 762 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationPlanner::executePlan().
|
inlineprotectedvirtual |
Allow subclasses to override and print debug traces before/after vplan execution, when trace information is requested.
Reimplemented in llvm::EpilogueVectorizerEpilogueLoop, and llvm::EpilogueVectorizerMainLoop.
Definition at line 761 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationPlanner::executePlan().
|
protected |
If there is a cast involved in the induction variable ID
, which should be ignored in the vectorized loop body, this function records the VectorLoopValue of the respective Phi also as the VectorLoopValue of the cast.
We had already proved that the casted Phi is equal to the uncasted Phi in the vectorized loop (under a runtime guard), and therefore there is no need to vectorize the cast - the same value can be used in the vector loop for both the Phi and the cast. If VectorLoopValue
is a scalarized value, Lane
is also specified, Otherwise, VectorLoopValue
is a widened/vectorized value.
EntryVal
is the value from the original loop that maps to the vector phi node and is used to distinguish what is the IV currently being processed - original one (if EntryVal
is a phi corresponding to the original IV) or the "newly-created" one based on the proof mentioned above (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the latter case EntryVal
is a TruncInst and we must not record anything for that IV, but it's error-prone to expect callers of this routine to care about that, hence this explicit parameter.
Definition at line 2326 of file LoopVectorize.cpp.
References assert(), and llvm::VPTransformState::set().
Referenced by buildScalarSteps().
Generate a shuffle sequence that will reverse the vector Vec.
Definition at line 2612 of file LoopVectorize.cpp.
References assert(), Builder, llvm::IRBuilderBase::CreateVectorReverse(), llvm::Value::getType(), and llvm::Type::isVectorTy().
Referenced by vectorizeInterleaveGroup(), and vectorizeMemoryInstruction().
void InnerLoopVectorizer::scalarizeInstruction | ( | Instruction * | Instr, |
VPValue * | Def, | ||
VPUser & | Operands, | ||
const VPIteration & | Instance, | ||
bool | IfPredicateInstr, | ||
VPTransformState & | State | ||
) |
A helper function to scalarize a single Instruction in the innermost loop.
Generates a sequence of scalar instances for each lane between MinLane
and MaxLane
, times each part between MinPart
and MaxPart
, inclusive. Uses the VPValue operands from Operands
instead of Instr's
operands.
Definition at line 2980 of file LoopVectorize.cpp.
References AC, addNewMetadata(), assert(), llvm::VPTransformState::Builder, Builder, llvm::Instruction::clone(), llvm::LoopBase< BlockT, LoopT >::contains(), Cost, llvm::tgtok::Def, llvm::numbers::e, llvm::VPTransformState::get(), llvm::VPLane::getFirstLane(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::GetInsertPoint(), llvm::Value::getName(), llvm::User::getNumOperands(), llvm::User::getOperand(), llvm::Value::getType(), llvm::IRBuilderBase::Insert(), llvm::Type::isAggregateType(), llvm::VPIteration::isFirstIteration(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::Type::isVoidTy(), llvm::VPIteration::Lane, op, OrigLoop, PredicatedInstructions, llvm::AssumptionCache::registerAssumption(), llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::User::setOperand(), and llvm::VPTransformState::VF.
Referenced by llvm::VPReplicateRecipe::execute().
Set the debug location in the builder using the debug location in the instruction.
Definition at line 1047 of file LoopVectorize.cpp.
References assert(), B, llvm::DILocation::cloneByMultiplyingDuplicationFactor(), llvm::dbgs(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LinearPolySize< LeafTy >::isScalable(), LLVM_DEBUG, UF, and VF.
Referenced by llvm::VPBlendRecipe::execute(), fixReduction(), scalarizeInstruction(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().
|
protected |
Returns true if an instruction I
should be scalarized instead of vectorized for the chosen vectorization factor.
Definition at line 2311 of file LoopVectorize.cpp.
References Cost, I, llvm::LoopVectorizationCostModel::isProfitableToScalarize(), llvm::LoopVectorizationCostModel::isScalarAfterVectorization(), and VF.
Referenced by needsScalarInduction().
|
protected |
Iteratively sink the scalarized operands of a predicated instruction into the block that was created for it.
Definition at line 4509 of file LoopVectorize.cpp.
References llvm::all_of(), BB, llvm::SmallVectorImpl< T >::clear(), llvm::SetVector< T, Vector, Set >::empty(), llvm::PHINode::getIncomingValueNumForOperand(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::Instruction::getParent(), I, llvm::SetVector< T, Vector, Set >::insert(), LI, llvm::User::op_begin(), llvm::User::op_end(), and llvm::SetVector< T, Vector, Set >::pop_back_val().
|
protected |
Shrinks vector element sizes to the smallest bitwidth they can be legally represented as.
Definition at line 3872 of file LoopVectorize.cpp.
References B, Cost, llvm::SmallPtrSetImpl< PtrType >::count(), llvm::tgtok::Def, llvm::Instruction::eraseFromParent(), llvm::IntegerType::get(), llvm::VPTransformState::get(), llvm::FixedVectorType::get(), llvm::Type::getContext(), llvm::LoopVectorizationCostModel::getMinimalBitwidths(), llvm::User::getOperand(), llvm::VPlan::getVPValue(), llvm::VPTransformState::hasAnyVectorValue(), I, llvm::ARM_PROC::IE, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm_unreachable, llvm::VPTransformState::Plan, llvm::VPTransformState::reset(), SI, smallestIntegerVectorType(), llvm::Value::takeName(), UF, and llvm::Value::use_empty().
Referenced by fixVectorizedLoop().
void InnerLoopVectorizer::vectorizeInterleaveGroup | ( | const InterleaveGroup< Instruction > * | Group, |
ArrayRef< VPValue * > | VPDefs, | ||
VPTransformState & | State, | ||
VPValue * | Addr, | ||
ArrayRef< VPValue * > | StoredValues, | ||
VPValue * | BlockInMask = nullptr |
||
) |
Try to vectorize interleaved access group Group
with the base address given in Addr
, optionally masking the vector operations if BlockInMask
is non-null.
Use State
to translate given VPValues to IR values in the vectorized loop.
Definition at line 2656 of file LoopVectorize.cpp.
References llvm::InterleaveGroup< InstTy >::addMetadata(), Addr, assert(), Builder, llvm::concatenateVectors(), Cost, llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateBitCast(), llvm::createBitMaskForGaps(), createBitOrPointerCast(), llvm::IRBuilderBase::CreateGEP(), llvm::createInterleaveMask(), llvm::IRBuilderBase::CreateMaskedLoad(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::createReplicatedMask(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::createStrideMask(), DL, gep, llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::PoisonValue::get(), llvm::InterleaveGroup< InstTy >::getAlign(), llvm::Module::getDataLayout(), llvm::InterleaveGroup< InstTy >::getFactor(), llvm::InterleaveGroup< InstTy >::getIndex(), llvm::InterleaveGroup< InstTy >::getInsertPos(), llvm::IRBuilderBase::getInt32(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::InterleaveGroup< InstTy >::getMember(), getMemInstValueType(), llvm::Instruction::getModule(), llvm::Type::getPointerAddressSpace(), llvm::Type::getPointerTo(), llvm::Value::getType(), i, I, Index, llvm::InterleaveGroup< InstTy >::isReverse(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::LoopVectorizationCostModel::isScalarEpilogueAllowed(), llvm::InterleaveGroup< InstTy >::requiresScalarEpilogue(), reverseVector(), llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::Value::stripPointerCasts(), UF, useMaskedInterleavedAccesses(), and VF.
Referenced by llvm::VPInterleaveRecipe::execute().
void InnerLoopVectorizer::vectorizeMemoryInstruction | ( | Instruction * | Instr, |
VPTransformState & | State, | ||
VPValue * | Def, | ||
VPValue * | Addr, | ||
VPValue * | StoredValue, | ||
VPValue * | BlockInMask | ||
) |
Vectorize Load and Store instructions with the base address given in Addr
, optionally masking the vector operations if BlockInMask
is non-null.
Use State
to translate given VPValues to IR values in the vectorized loop.
Definition at line 2838 of file LoopVectorize.cpp.
References addMetadata(), Addr, assert(), Builder, llvm::LoopVectorizationCostModel::CM_GatherScatter, llvm::LoopVectorizationCostModel::CM_Widen, llvm::LoopVectorizationCostModel::CM_Widen_Reverse, Cost, llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedGather(), llvm::IRBuilderBase::CreateMaskedLoad(), llvm::IRBuilderBase::CreateMaskedScatter(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::IRBuilderBase::CreateMul(), createStepForVF(), llvm::IRBuilderBase::CreateSub(), llvm::tgtok::Def, gep, llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::PoisonValue::get(), llvm::IRBuilderBase::getInt32(), llvm::IRBuilderBase::getInt32Ty(), llvm::getLoadStoreAlignment(), getMemInstValueType(), llvm::getRuntimeVF(), llvm::LoopVectorizationCostModel::getWideningDecision(), LI, reverseVector(), llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::GetElementPtrInst::setIsInBounds(), SI, UF, and VF.
Referenced by llvm::VPWidenMemoryInstructionRecipe::execute().
void InnerLoopVectorizer::widenCallInstruction | ( | CallInst & | I, |
VPValue * | Def, | ||
VPUser & | ArgOperands, | ||
VPTransformState & | State | ||
) |
Widen a single call instruction within the innermost loop.
Definition at line 4969 of file LoopVectorize.cpp.
References addMetadata(), Arg, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), Builder, llvm::Instruction::copyFastMathFlags(), Cost, llvm::IRBuilderBase::CreateCall(), llvm::tgtok::Def, llvm::enumerate(), llvm::VFShape::get(), llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopVectorizationCostModel::getVectorCallCost(), llvm::LoopVectorizationCostModel::getVectorIntrinsicCost(), llvm::getVectorIntrinsicIDForCall(), llvm::VFDatabase::getVectorizedFunction(), llvm::hasVectorInstrinsicScalarOpd(), I, llvm::InstructionCost::isValid(), llvm::ElementCount::isVector(), M, llvm::VPUser::operands(), llvm::VPTransformState::set(), setDebugLocFromInst(), TLI, llvm::ToVectorTy(), UF, and VF.
Referenced by llvm::VPWidenCallRecipe::execute().
void InnerLoopVectorizer::widenGEP | ( | GetElementPtrInst * | GEP, |
VPValue * | VPDef, | ||
VPUser & | Indices, | ||
unsigned | UF, | ||
ElementCount | VF, | ||
bool | IsPtrLoopInvariant, | ||
SmallBitVector & | IsIndexLoopInvariant, | ||
VPTransformState & | State | ||
) |
Vectorize a single GetElementPtrInst based on information gathered and decisions taken during planning.
Definition at line 4588 of file LoopVectorize.cpp.
References addMetadata(), llvm::SmallBitVector::all(), assert(), Builder, llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreateVectorSplat(), E, GEP, llvm::VPTransformState::get(), I, llvm::IRBuilderBase::Insert(), llvm::ElementCount::isScalar(), llvm::ElementCount::isVector(), Operands, llvm::VPTransformState::set(), UF, and VF.
Referenced by llvm::VPWidenGEPRecipe::execute().
void InnerLoopVectorizer::widenInstruction | ( | Instruction & | I, |
VPValue * | Def, | ||
VPUser & | Operands, | ||
VPTransformState & | State | ||
) |
Widen a single instruction within the innermost loop.
Vectorize casts.
Definition at line 4861 of file LoopVectorize.cpp.
References llvm::MCID::Add, addMetadata(), B, Builder, llvm::MCID::Call, llvm::IRBuilderBase::CreateCast(), llvm::IRBuilderBase::CreateFCmp(), llvm::IRBuilderBase::CreateICmp(), llvm::IRBuilderBase::CreateNAryOp(), llvm::dbgs(), llvm::tgtok::Def, llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::User::getOperand(), I, llvm::ElementCount::isScalar(), LLVM_DEBUG, llvm_unreachable, llvm::User::operands(), llvm::MCID::Select, llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::IRBuilderBase::setFastMathFlags(), UF, and VF.
Referenced by llvm::VPWidenRecipe::execute().
void InnerLoopVectorizer::widenIntOrFpInduction | ( | PHINode * | IV, |
Value * | Start, | ||
TruncInst * | Trunc, | ||
VPValue * | Def, | ||
VPValue * | CastDef, | ||
VPTransformState & | State | ||
) |
Widen an integer or floating-point induction variable IV
.
If Trunc
is provided, the integer induction variable will first be truncated to the corresponding type.
Definition at line 2354 of file LoopVectorize.cpp.
Referenced by llvm::VPWidenIntOrFpInductionRecipe::execute().
void InnerLoopVectorizer::widenPHIInstruction | ( | Instruction * | PN, |
RecurrenceDescriptor * | RdxDesc, | ||
VPWidenPHIRecipe * | PhiR, | ||
VPTransformState & | State | ||
) |
Vectorize a single PHINode in a block.
This method handles the induction variable canonicalization. It supports both VF = 1 for unrolled loops and arbitrary length vectors.
Definition at line 4659 of file LoopVectorize.cpp.
References llvm::PHINode::addIncoming(), assert(), Builder, Cost, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::count(), llvm::GetElementPtrInst::Create(), llvm::PHINode::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateMul(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateSExtOrTrunc(), llvm::IRBuilderBase::CreateStepVector(), llvm::IRBuilderBase::CreateVectorSplat(), DL, emitTransformedIndex(), EnableVPlanNativePath, GEP, llvm::ConstantInt::get(), llvm::VectorType::get(), llvm::Module::getDataLayout(), llvm::RecurrenceDescriptor::getFastMathFlags(), llvm::BasicBlock::getFirstInsertionPt(), llvm::LinearPolySize< LeafTy >::getFixedValue(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::getInductionVars(), llvm::IRBuilderBase::getInt32(), llvm::InductionDescriptor::getKind(), llvm::VPValue::getLiveInIRValue(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::BasicBlock::getModule(), llvm::User::getNumOperands(), llvm::Instruction::getParent(), llvm::Type::getPointerElementType(), llvm::RecurrenceDescriptor::getRecurrenceIdentity(), llvm::RecurrenceDescriptor::getRecurrenceKind(), llvm::getRuntimeVF(), llvm::Type::getScalarType(), llvm::PredicatedScalarEvolution::getSE(), llvm::ConstantVector::getSplat(), llvm::InductionDescriptor::getStartValue(), llvm::VPWidenPHIRecipe::getStartValue(), llvm::InductionDescriptor::getStep(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, Induction, llvm::LoopVectorizationLegality::isFirstOrderRecurrence(), llvm::LoopVectorizationCostModel::isInLoopReduction(), llvm::RecurrenceDescriptor::isMinMaxRecurrenceKind(), llvm::LoopVectorizationLegality::isReductionVariable(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isScalar(), llvm::LoopVectorizationCostModel::isScalarAfterVectorization(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), Legal, LI, llvm_unreachable, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::lookup(), LoopVectorBody, LoopVectorPreHeader, OrigLoop, OrigPHIsToFix, P, PSE, llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::VPTransformState::UF, UF, llvm::VPTransformState::VF, and VF.
Referenced by llvm::VPWidenPHIRecipe::execute().
void InnerLoopVectorizer::widenSelectInstruction | ( | SelectInst & | I, |
VPValue * | VPDef, | ||
VPUser & | Operands, | ||
bool | InvariantCond, | ||
VPTransformState & | State | ||
) |
Widen a single select instruction within the innermost loop.
Definition at line 5039 of file LoopVectorize.cpp.
References addMetadata(), Builder, Cond, llvm::IRBuilderBase::CreateSelect(), llvm::VPTransformState::get(), I, Operands, llvm::VPTransformState::set(), setDebugLocFromInst(), and UF.
Referenced by llvm::VPWidenSelectRecipe::execute().
|
friend |
Definition at line 573 of file LoopVectorize.cpp.
|
protected |
Alias Analysis.
Definition at line 779 of file LoopVectorize.cpp.
|
protected |
Assumption Cache.
Definition at line 788 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationCostModel::collectValuesToIgnore(), getScalarEpilogueLowering(), processLoopInVPlanNativePath(), and scalarizeInstruction().
|
protected |
Definition at line 857 of file LoopVectorize.cpp.
Referenced by areSafetyChecksAdded(), emitMemRuntimeChecks(), and emitSCEVChecks().
|
protected |
BFI and PSI are used to check for profile guided size optimizations.
Definition at line 868 of file LoopVectorize.cpp.
Referenced by getScalarEpilogueLowering(), InnerLoopVectorizer(), and processLoopInVPlanNativePath().
|
protected |
The builder that we use.
Definition at line 809 of file LoopVectorize.cpp.
Referenced by buildScalarSteps(), completeLoopSkeleton(), createBitOrPointerCast(), llvm::VPRecipeBuilder::createBlockInMask(), llvm::VPRecipeBuilder::createEdgeMask(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), llvm::LoopVectorizationPlanner::executePlan(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixNonInductionPHIs(), fixReduction(), llvm::VPTransformState::get(), getBroadcastInstrs(), getOrCreateTripCount(), getOrCreateVectorTripCount(), getStepVector(), packScalarIntoVectorValue(), reverseVector(), scalarizeInstruction(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().
|
protected |
The profitablity analysis.
Definition at line 854 of file LoopVectorize.cpp.
Referenced by buildScalarSteps(), completeLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), fixLCSSAPHIs(), fixReduction(), getOrCreateVectorTripCount(), llvm::LoopVectorizationCostModel::getVectorCallCost(), scalarizeInstruction(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::LoopVectorizationCostModel::setWideningDecision(), shouldScalarizeInstruction(), truncateToMinimalBitwidths(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), and widenPHIInstruction().
|
protected |
Dominator Tree.
Definition at line 776 of file LoopVectorize.cpp.
Referenced by completeLoopSkeleton(), GeneratedRTChecks::Create(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), GeneratedRTChecks::emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), GeneratedRTChecks::emitSCEVChecks(), llvm::LoopVectorizationPlanner::executePlan(), getBroadcastInstrs(), getScalarEpilogueLowering(), processLoopInVPlanNativePath(), and GeneratedRTChecks::~GeneratedRTChecks().
|
protected |
The new Induction variable which was added to the new block.
Definition at line 836 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), llvm::LoopVectorizationCostModel::collectValuesToIgnore(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), llvm::LoopVectorizationPlanner::executePlan(), llvm::EpilogueVectorizerMainLoop::printDebugTracesAtEnd(), llvm::EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd(), and widenPHIInstruction().
Definition at line 861 of file LoopVectorize.cpp.
Referenced by createInductionResumeValues().
|
protected |
The legality analysis.
Definition at line 851 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationCostModel::blockNeedsPredication(), llvm::LoopVectorizationCostModel::canVectorizeReductions(), llvm::LoopVectorizationCostModel::collectInLoopReductions(), llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), llvm::LoopVectorizationCostModel::collectValuesToIgnore(), llvm::LoopVectorizationCostModel::computeMaxVF(), llvm::VPRecipeBuilder::createBlockInMask(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), createVectorizedLoopSkeleton(), emitMemRuntimeChecks(), fixCrossIterationPHIs(), fixReduction(), getAddressAccessSCEV(), getOrCreateTripCount(), llvm::LoopVectorizationCostModel::getSmallestAndWidestTypes(), llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationCostModel::isLegalMaskedLoad(), llvm::LoopVectorizationCostModel::isLegalMaskedStore(), llvm::LoopVectorizationCostModel::isOptimizableIVTruncate(), llvm::LoopVectorizationCostModel::isPredicatedInst(), llvm::LoopVectorizationCostModel::isScalarWithPredication(), isStrideMul(), llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened(), llvm::LoopVectorizationCostModel::runtimeChecksRequired(), llvm::LoopVectorizationCostModel::selectInterleaveCount(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::VPRecipeBuilder::tryToCreateWidenRecipe(), and widenPHIInstruction().
|
protected |
Loop Info.
Definition at line 773 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationCostModel::calculateRegisterUsage(), collectSupportedLoops(), completeLoopSkeleton(), GeneratedRTChecks::Create(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), GeneratedRTChecks::emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), GeneratedRTChecks::emitSCEVChecks(), emitTransformedIndex(), llvm::LoopVectorizationPlanner::executePlan(), fixFirstOrderRecurrence(), fixReduction(), getScalarEpilogueLowering(), llvm::LoopVectorizationCostModel::isLegalGatherOrScatter(), llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened(), processLoopInVPlanNativePath(), sinkScalarOperands(), vectorizeMemoryInstruction(), and widenPHIInstruction().
|
protected |
A list of all bypass blocks. The first block is the entry of the loop.
Definition at line 833 of file LoopVectorize.cpp.
Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), and fixReduction().
|
protected |
The (unique) ExitBlock of the scalar loop.
Note that there can be multiple exiting edges reaching this block.
Definition at line 824 of file LoopVectorize.cpp.
Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), emitSCEVChecks(), fixLCSSAPHIs(), and fixReduction().
|
protected |
Middle Block between the vector and the scalar.
Definition at line 820 of file LoopVectorize.cpp.
Referenced by completeLoopSkeleton(), createInductionResumeValues(), createVectorLoopSkeleton(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), and fixReduction().
|
protected |
The scalar loop body.
Definition at line 830 of file LoopVectorize.cpp.
Referenced by createInductionResumeValues(), and createVectorLoopSkeleton().
|
protected |
The scalar-loop preheader.
Definition at line 817 of file LoopVectorize.cpp.
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), fixFirstOrderRecurrence(), and fixReduction().
|
protected |
The vector loop body.
Definition at line 827 of file LoopVectorize.cpp.
Referenced by completeLoopSkeleton(), createVectorLoopSkeleton(), emitTransformedIndex(), fixFirstOrderRecurrence(), fixReduction(), and widenPHIInstruction().
|
protected |
The vector-loop preheader.
Definition at line 814 of file LoopVectorize.cpp.
Referenced by completeLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), GeneratedRTChecks::emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), GeneratedRTChecks::emitSCEVChecks(), fixFirstOrderRecurrence(), getBroadcastInstrs(), and widenPHIInstruction().
|
protected |
It's only set up (non-null) if memchecks were used.
This is currently only used to add no-alias metadata based on the memchecks. The actually versioning is performed manually.
Definition at line 798 of file LoopVectorize.cpp.
Referenced by addNewMetadata(), and emitMemRuntimeChecks().
|
protected |
The induction variable of the old basic block.
Definition at line 839 of file LoopVectorize.cpp.
Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), and createVectorizedLoopSkeleton().
|
protected |
Definition at line 873 of file LoopVectorize.cpp.
Referenced by emitMemRuntimeChecks(), emitSCEVChecks(), and InnerLoopVectorizer().
|
protected |
Interface to emit optimization remarks.
Definition at line 791 of file LoopVectorize.cpp.
Referenced by checkMixedPrecision(), collectSupportedLoops(), completeLoopSkeleton(), llvm::LoopVectorizationCostModel::computeMaxVF(), emitMemRuntimeChecks(), isExplicitVecOuterLoop(), llvm::LoopVectorizationPlanner::plan(), processLoopInVPlanNativePath(), llvm::reportVectorizationFailure(), llvm::LoopVectorizationCostModel::runtimeChecksRequired(), and llvm::LoopVectorizationCostModel::selectVectorizationFactor().
|
protected |
The original loop.
Definition at line 762 of file LoopVectorize.cpp.
Referenced by clearReductionWrapFlags(), llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), completeLoopSkeleton(), llvm::VPRecipeBuilder::createBlockInMask(), llvm::VPRecipeBuilder::createEdgeMask(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), fixCrossIterationPHIs(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixReduction(), fixupIVUsers(), fixVectorizedLoop(), getBroadcastInstrs(), InnerLoopVectorizer(), needsScalarInduction(), llvm::LoopVectorizationPlanner::plan(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), scalarizeInstruction(), llvm::VPRecipeBuilder::tryToCreateWidenRecipe(), and widenPHIInstruction().
|
protected |
Definition at line 865 of file LoopVectorize.cpp.
Referenced by fixNonInductionPHIs(), fixVectorizedLoop(), and widenPHIInstruction().
|
protected |
Store instructions that were predicated.
Definition at line 842 of file LoopVectorize.cpp.
Referenced by scalarizeInstruction().
|
protected |
A wrapper around ScalarEvolution used to add runtime SCEV checks.
Applies dynamic knowledge to simplify SCEV expressions and converts them to a more usable form.
Definition at line 770 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), createInductionResumeValues(), emitMemRuntimeChecks(), fixupIVUsers(), fixVectorizedLoop(), getAddressAccessSCEV(), getOrCreateTripCount(), processLoopInVPlanNativePath(), llvm::LoopVectorizationCostModel::runtimeChecksRequired(), llvm::LoopVectorizationCostModel::selectInterleaveCount(), llvm::VPRecipeBuilder::tryToCreateWidenRecipe(), and widenPHIInstruction().
|
protected |
Definition at line 869 of file LoopVectorize.cpp.
Referenced by getScalarEpilogueLowering(), InnerLoopVectorizer(), and processLoopInVPlanNativePath().
|
protected |
Structure to hold information about generated runtime checks, responsible for cleaning the checks, if vectorization turns out unprofitable.
Definition at line 877 of file LoopVectorize.cpp.
Referenced by emitMemRuntimeChecks(), and emitSCEVChecks().
|
protected |
Target Library Info.
Definition at line 782 of file LoopVectorize.cpp.
Referenced by getScalarEpilogueLowering(), llvm::LoopVectorizationCostModel::getVectorCallCost(), llvm::LoopVectorizationCostModel::getVectorIntrinsicCost(), processLoopInVPlanNativePath(), and widenCallInstruction().
|
protected |
Trip count of the original loop.
Definition at line 845 of file LoopVectorize.cpp.
Referenced by getOrCreateTripCount().
|
protected |
Target Transform Info.
Definition at line 785 of file LoopVectorize.cpp.
Referenced by llvm::LoopVectorizationCostModel::calculateRegisterUsage().
|
protected |
The vectorization unroll factor to use.
Each scalar is vectorized to this many different vector instructions.
Definition at line 806 of file LoopVectorize.cpp.
Referenced by buildScalarSteps(), clearReductionWrapFlags(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixReduction(), getOrCreateVectorTripCount(), llvm::LoopVectorizationPlanner::setBestPlan(), setDebugLocFromInst(), truncateToMinimalBitwidths(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().
|
protected |
Trip count of the widened loop (TripCount - TripCount % (VF*UF))
Definition at line 848 of file LoopVectorize.cpp.
Referenced by completeLoopSkeleton(), createInductionResumeValues(), and getOrCreateVectorTripCount().
|
protected |
The vectorization SIMD factor to use.
Each vector will have this many vector elements.
Definition at line 802 of file LoopVectorize.cpp.
Referenced by buildScalarSteps(), llvm::LoopVectorizationPlanner::buildVPlans(), llvm::LoopVectorizationCostModel::calculateRegisterUsage(), llvm::LoopVectorizationCostModel::canTruncateToMinimalBitwidth(), llvm::LoopVectorizationCostModel::collectInstsToScalarize(), llvm::LoopVectorizationCostModel::collectUniformsAndScalars(), createBitOrPointerCast(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createStepForVF(), createVectorizedLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixReduction(), fixVectorizedLoop(), llvm::VPTransformState::get(), getBroadcastInstrs(), getOrCreateVectorTripCount(), llvm::getRuntimeVF(), llvm::LoopVectorizationCostModel::getVectorCallCost(), llvm::LoopVectorizationCostModel::getVectorIntrinsicCost(), llvm::LoopVectorizationCostModel::getWideningCost(), llvm::LoopVectorizationCostModel::getWideningDecision(), llvm::VPRecipeBuilder::handleReplication(), llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationCostModel::isOptimizableIVTruncate(), llvm::LoopVectorizationCostModel::isProfitableToScalarize(), llvm::LoopVectorizationCostModel::isScalarAfterVectorization(), llvm::LoopVectorizationCostModel::isScalarWithPredication(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), MaybeVectorizeType(), packScalarIntoVectorValue(), llvm::LoopVectorizationPlanner::plan(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), processLoopInVPlanNativePath(), llvm::LoopVectorizationCostModel::selectInterleaveCount(), llvm::LoopVectorizationPlanner::setBestPlan(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), setDebugLocFromInst(), llvm::LoopVectorizationCostModel::setWideningDecision(), shouldScalarizeInstruction(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), and widenPHIInstruction().