InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...

Inheritance diagram for llvm::InnerLoopVectorizer:

[legend]

Collaboration diagram for llvm::InnerLoopVectorizer:

[legend]

Public Types
using	VectorParts = SmallVector< Value *, 2 >
	A type for vectorized values in the new loop. More...

Public Member Functions
	InnerLoopVectorizer (Loop OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo LI, DominatorTree DT, const TargetLibraryInfo TLI, const TargetTransformInfo TTI, AssumptionCache AC, OptimizationRemarkEmitter ORE, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationLegality LVL, LoopVectorizationCostModel CM, BlockFrequencyInfo BFI, ProfileSummaryInfo *PSI)

virtual	~InnerLoopVectorizer ()=default

virtual BasicBlock *	createVectorizedLoopSkeleton ()
	Create a new empty loop that will contain vectorized instructions later on, while the old loop will be used as the scalar remainder. More...

void	widenInstruction (Instruction &I, VPValue *Def, VPUser &Operands, VPTransformState &State)
	Widen a single instruction within the innermost loop. More...

void	widenCallInstruction (CallInst &I, VPValue *Def, VPUser &ArgOperands, VPTransformState &State)
	Widen a single call instruction within the innermost loop. More...

void	widenSelectInstruction (SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State)
	Widen a single select instruction within the innermost loop. More...

void	fixVectorizedLoop ()
	Fix the vectorized code, taking care of header phi's, live-outs, and more. More...

bool	areSafetyChecksAdded ()

void	widenGEP (GetElementPtrInst GEP, VPValue VPDef, VPUser &Indices, unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State)
	Vectorize a single GetElementPtrInst based on information gathered and decisions taken during planning. More...

void	widenPHIInstruction (Instruction PN, RecurrenceDescriptor RdxDesc, Value *StartV, unsigned UF, ElementCount VF)
	Vectorize a single PHINode in a block. More...

void	scalarizeInstruction (Instruction *Instr, VPUser &Operands, const VPIteration &Instance, bool IfPredicateInstr, VPTransformState &State)
	A helper function to scalarize a single Instruction in the innermost loop. More...

void	widenIntOrFpInduction (PHINode IV, Value Start, TruncInst *Trunc=nullptr)
	Widen an integer or floating-point induction variable `IV`. More...

Value *	getOrCreateVectorValue (Value *V, unsigned Part)
	getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a vector or scalar value on-demand if one is not yet available. More...

void	setVectorValue (Value Scalar, unsigned Part, Value Vector)

Value *	getOrCreateScalarValue (Value *V, const VPIteration &Instance)
	Return a value in the new loop corresponding to `V` from the original loop at unroll and vector indices `Instance`. More...

void	packScalarIntoVectorValue (Value *V, const VPIteration &Instance)
	Construct the vector value of a scalarized value `V` one lane at a time. More...

void	vectorizeInterleaveGroup (const InterleaveGroup< Instruction > Group, ArrayRef< VPValue > VPDefs, VPTransformState &State, VPValue Addr, ArrayRef< VPValue > StoredValues, VPValue *BlockInMask=nullptr)
	Try to vectorize interleaved access group `Group` with the base address given in `Addr`, optionally masking the vector operations if `BlockInMask` is non-null. More...

void	vectorizeMemoryInstruction (Instruction Instr, VPTransformState &State, VPValue Def, VPValue Addr, VPValue StoredValue, VPValue *BlockInMask)
	Vectorize Load and Store instructions with the base address given in `Addr`, optionally masking the vector operations if `BlockInMask` is non-null. More...

void	setDebugLocFromInst (IRBuilder<> &B, const Value *Ptr)
	Set the debug location in the builder using the debug location in the instruction. More...

void	fixNonInductionPHIs (void)
	Fix the non-induction PHIs in the OrigPHIsToFix vector. More...

Protected Types
using	PhiVector = SmallVector< PHINode *, 4 >
	A small list of PHINodes. More...

using	ScalarParts = SmallVector< SmallVector< Value *, 4 >, 2 >
	A type for scalarized values in the new loop. More...

Protected Member Functions
void	fixupIVUsers (PHINode OrigPhi, const InductionDescriptor &II, Value CountRoundDown, Value EndValue, BasicBlock MiddleBlock)
	Set up the values of the IVs correctly when exiting the vector loop. More...

PHINode *	createInductionVariable (Loop L, Value Start, Value End, Value Step, Instruction *DL)
	Create a new induction variable inside L. More...

void	fixCrossIterationPHIs ()
	Handle all cross-iteration phis in the header. More...

void	fixFirstOrderRecurrence (PHINode *Phi)
	Fix a first-order recurrence. More...

void	fixReduction (PHINode *Phi)
	Fix a reduction cross-iteration phi. More...

void	clearReductionWrapFlags (RecurrenceDescriptor &RdxDesc)
	Clear NSW/NUW flags from reduction instructions if necessary. More...

void	fixLCSSAPHIs ()
	Fixup the LCSSA phi nodes in the unique exit block. More...

void	sinkScalarOperands (Instruction *PredInst)
	Iteratively sink the scalarized operands of a predicated instruction into the block that was created for it. More...

void	truncateToMinimalBitwidths ()
	Shrinks vector element sizes to the smallest bitwidth they can be legally represented as. More...

virtual Value *	getBroadcastInstrs (Value *V)
	Create a broadcast instruction. More...

virtual Value *	getStepVector (Value Val, int StartIdx, Value Step, Instruction::BinaryOps Opcode=Instruction::BinaryOpsEnd)
	This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...) to each vector element of Val. More...

void	buildScalarSteps (Value ScalarIV, Value Step, Instruction *EntryVal, const InductionDescriptor &ID)
	Compute scalar induction steps. More...

void	createVectorIntOrFpInductionPHI (const InductionDescriptor &II, Value Step, Value Start, Instruction *EntryVal)
	Create a vector induction phi node based on an existing scalar one. More...

bool	shouldScalarizeInstruction (Instruction *I) const
	Returns true if an instruction `I` should be scalarized instead of vectorized for the chosen vectorization factor. More...

bool	needsScalarInduction (Instruction *IV) const
	Returns true if we should generate a scalar version of `IV`. More...

void	recordVectorLoopValueForInductionCast (const InductionDescriptor &ID, const Instruction EntryVal, Value VectorLoopValue, unsigned Part, unsigned Lane=UINT_MAX)
	If there is a cast involved in the induction variable `ID`, which should be ignored in the vectorized loop body, this function records the VectorLoopValue of the respective Phi also as the VectorLoopValue of the cast. More...

virtual Value *	reverseVector (Value *Vec)
	Generate a shuffle sequence that will reverse the vector Vec. More...

Value *	getOrCreateTripCount (Loop *NewLoop)
	Returns (and creates if needed) the original loop trip count. More...

Value *	getOrCreateVectorTripCount (Loop *NewLoop)
	Returns (and creates if needed) the trip count of the widened loop. More...

Value *	createBitOrPointerCast (Value V, VectorType DstVTy, const DataLayout &DL)
	Returns a bitcasted value to the requested vector type. More...

void	emitMinimumIterationCountCheck (Loop L, BasicBlock Bypass)
	Emit a bypass check to see if the vector trip count is zero, including if it overflows. More...

void	emitSCEVChecks (Loop L, BasicBlock Bypass)
	Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct. More...

void	emitMemRuntimeChecks (Loop L, BasicBlock Bypass)
	Emit bypass checks to check any memory assumptions we may have made. More...

Value *	emitTransformedIndex (IRBuilder<> &B, Value Index, ScalarEvolution SE, const DataLayout &DL, const InductionDescriptor &ID) const
	Compute the transformed value of Index at offset StartValue using step StepValue. More...

Loop *	createVectorLoopSkeleton (StringRef Prefix)
	Emit basic blocks (prefixed with `Prefix`) for the iteration check, vector loop preheader, middle block and scalar preheader. More...

void	createInductionResumeValues (Loop L, Value VectorTripCount, std::pair< BasicBlock , Value > AdditionalBypass={nullptr, nullptr})
	Create new phi nodes for the induction variables to resume iteration count in the scalar epilogue, from where the vectorized loop left off (given by `VectorTripCount`). More...

BasicBlock *	completeLoopSkeleton (Loop L, MDNode OrigLoopID)
	Complete the loop skeleton by adding debug MDs, creating appropriate conditional branches in the middle block, preparing the builder and running the verifier. More...

void	addNewMetadata (Instruction To, const Instruction Orig)
	Add additional metadata to `To` that was not present on `Orig`. More...

void	addMetadata (Instruction To, Instruction From)
	Add metadata from one instruction to another. More...

void	addMetadata (ArrayRef< Value * > To, Instruction *From)
	Similar to the previous function but it adds the metadata to a vector of instructions. More...

virtual void	printDebugTracesAtStart ()
	Allow subclasses to override and print debug traces before/after vplan execution, when trace information is requested. More...

virtual void	printDebugTracesAtEnd ()

Protected Attributes
Loop *	OrigLoop
	The original loop. More...

PredicatedScalarEvolution &	PSE
	A wrapper around ScalarEvolution used to add runtime SCEV checks. More...

LoopInfo *	LI
	Loop Info. More...

DominatorTree *	DT
	Dominator Tree. More...

AAResults *	AA
	Alias Analysis. More...

const TargetLibraryInfo *	TLI
	Target Library Info. More...

const TargetTransformInfo *	TTI
	Target Transform Info. More...

AssumptionCache *	AC
	Assumption Cache. More...

OptimizationRemarkEmitter *	ORE
	Interface to emit optimization remarks. More...

std::unique_ptr< LoopVersioning >	LVer
	LoopVersioning. More...

ElementCount	VF
	The vectorization SIMD factor to use. More...

unsigned	UF
	The vectorization unroll factor to use. More...

IRBuilder	Builder
	The builder that we use. More...

BasicBlock *	LoopVectorPreHeader
	The vector-loop preheader. More...

BasicBlock *	LoopScalarPreHeader
	The scalar-loop preheader. More...

BasicBlock *	LoopMiddleBlock
	Middle Block between the vector and the scalar. More...

BasicBlock *	LoopExitBlock
	The (unique) ExitBlock of the scalar loop. More...

BasicBlock *	LoopVectorBody
	The vector loop body. More...

BasicBlock *	LoopScalarBody
	The scalar loop body. More...

SmallVector< BasicBlock *, 4 >	LoopBypassBlocks
	A list of all bypass blocks. The first block is the entry of the loop. More...

PHINode *	Induction = nullptr
	The new Induction variable which was added to the new block. More...

PHINode *	OldInduction = nullptr
	The induction variable of the old basic block. More...

VectorizerValueMap	VectorLoopValueMap
	Maps values from the original loop to their corresponding values in the vectorized loop. More...

SmallVector< Instruction *, 4 >	PredicatedInstructions
	Store instructions that were predicated. More...

Value *	TripCount = nullptr
	Trip count of the original loop. More...

Value *	VectorTripCount = nullptr
	Trip count of the widened loop (TripCount - TripCount % (VF*UF)) More...

LoopVectorizationLegality *	Legal
	The legality analysis. More...

LoopVectorizationCostModel *	Cost
	The profitablity analysis. More...

bool	AddedSafetyChecks = false

DenseMap< PHINode , Value >	IVEndValues

SmallVector< PHINode *, 8 >	OrigPHIsToFix

BlockFrequencyInfo *	BFI
	BFI and PSI are used to check for profile guided size optimizations. More...

ProfileSummaryInfo *	PSI

bool	OptForSizeBasedOnProfile

Friends
class	LoopVectorizationPlanner

Detailed Description

InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF).

This class performs the widening of scalars into vectors, or multiple scalars. This class also implements the following features:

It inserts an epilogue loop for handling loops that don't have iteration counts that are known to be a multiple of the vectorization factor.
It handles the code generation for reduction variables.
Scalarization (implementation using scalars) of un-vectorizable instructions. InnerLoopVectorizer does not perform any vectorization-legality checks, and relies on the caller to check for the different legality aspects. The InnerLoopVectorizer relies on the LoopVectorizationLegality class to provide information about the induction and reduction variables that were found to a given vectorization factor.

Definition at line 458 of file LoopVectorize.cpp.

Member Typedef Documentation

◆ PhiVector

using llvm::InnerLoopVectorizer::PhiVector = SmallVector<PHINode *, 4>

protected

A small list of PHINodes.

Definition at line 605 of file LoopVectorize.cpp.

◆ ScalarParts

using llvm::InnerLoopVectorizer::ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>

protected

A type for scalarized values in the new loop.

Each value from the original loop, when scalarized, is represented by UF x VF scalar values in the new unrolled loop, where UF is the unroll factor and VF is the vectorization factor.

Definition at line 611 of file LoopVectorize.cpp.

◆ VectorParts

using llvm::InnerLoopVectorizer::VectorParts = SmallVector<Value *, 2>

A type for vectorized values in the new loop.

Each value from the original loop, when vectorized, is represented by UF vector values in the new unrolled loop, where UF is the unroll factor.

Definition at line 511 of file LoopVectorize.cpp.

Constructor & Destructor Documentation

◆ InnerLoopVectorizer()

llvm::InnerLoopVectorizer::InnerLoopVectorizer	(	Loop *	OrigLoop,
		PredicatedScalarEvolution &	PSE,
		LoopInfo *	LI,
		DominatorTree *	DT,
		const TargetLibraryInfo *	TLI,
		const TargetTransformInfo *	TTI,
		AssumptionCache *	AC,
		OptimizationRemarkEmitter *	ORE,
		ElementCount	VecWidth,
		unsigned	UnrollFactor,
		LoopVectorizationLegality *	LVL,
		LoopVectorizationCostModel *	CM,
		BlockFrequencyInfo *	BFI,
		ProfileSummaryInfo *	PSI
	)

inline

Definition at line 460 of file LoopVectorize.cpp.

References BFI, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::IRPass, OptForSizeBasedOnProfile, OrigLoop, PSI, and llvm::shouldOptimizeForSize().

◆ ~InnerLoopVectorizer()

virtual llvm::InnerLoopVectorizer::~InnerLoopVectorizer ( )

virtualdefault

Member Function Documentation

◆ addMetadata() [1/2]

void InnerLoopVectorizer::addMetadata	(	Instruction *	To,
		Instruction *	From
	)

protected

Add metadata from one instruction to another.

This includes both the original MDs from From and additional ones (

See also: addNewMetadata). Use this for newly created instructions in the vector loop.

Definition at line 1184 of file LoopVectorize.cpp.

References addNewMetadata(), From, and llvm::propagateMetadata().

Referenced by addMetadata(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), and widenSelectInstruction().

◆ addMetadata() [2/2]

void InnerLoopVectorizer::addMetadata	(	ArrayRef< Value * >	To,
		Instruction *	From
	)

protected

Similar to the previous function but it adds the metadata to a vector of instructions.

Definition at line 1190 of file LoopVectorize.cpp.

References addMetadata(), From, and I.

◆ addNewMetadata()

void InnerLoopVectorizer::addNewMetadata	(	Instruction *	To,
		const Instruction *	Orig
	)

protected

Add additional metadata to To that was not present on Orig.

Currently this is used to add the noalias annotations based on the inserted memchecks. Use this for instructions that are cloned into the vector loop.

Definition at line 1176 of file LoopVectorize.cpp.

References LVer.

Referenced by addMetadata(), and scalarizeInstruction().

◆ areSafetyChecksAdded()

bool llvm::InnerLoopVectorizer::areSafetyChecksAdded ( )

inline

Definition at line 506 of file LoopVectorize.cpp.

References AddedSafetyChecks.

Referenced by llvm::LoopVectorizePass::processLoop().

◆ buildScalarSteps()

void InnerLoopVectorizer::buildScalarSteps	(	Value *	ScalarIV,
		Value *	Step,
		Instruction *	EntryVal,
		const InductionDescriptor &	ID
	)

protected

Compute scalar induction steps.

ScalarIV is the scalar induction variable on which to base the steps, Step is the size of the step, and EntryVal is the value from the original loop that maps to the steps. Note that EntryVal doesn't have to be an induction variable - it can also be a truncate instruction.

Definition at line 2307 of file LoopVectorize.cpp.

References llvm::Add, addFastMathFlag(), assert(), Builder, Cost, llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateSIToFP(), createStepForVF(), llvm::FMul, llvm::IntegerType::get(), llvm::ConstantInt::get(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::Type::getScalarType(), getSignedIntOrFpConstant(), llvm::Value::getType(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::ElementCount::isVector(), llvm::Mul, recordVectorLoopValueForInductionCast(), llvm::VectorizerValueMap::setScalarValue(), UF, VectorLoopValueMap, and VF.

◆ clearReductionWrapFlags()

void InnerLoopVectorizer::clearReductionWrapFlags ( RecurrenceDescriptor & RdxDesc )

protected

Clear NSW/NUW flags from reduction instructions if necessary.

Definition at line 4338 of file LoopVectorize.cpp.

References llvm::Add, assert(), llvm::LoopBase< BlockT, LoopT >::contains(), llvm::SmallVectorBase< Size_T >::empty(), llvm::RecurrenceDescriptor::getLoopExitInstr(), getOrCreateVectorValue(), llvm::Instruction::getParent(), llvm::RecurrenceDescriptor::getRecurrenceKind(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::Mul, OrigLoop, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), UF, and llvm::Value::users().

Referenced by fixReduction().

◆ completeLoopSkeleton()

BasicBlock * InnerLoopVectorizer::completeLoopSkeleton	(	Loop *	L,
		MDNode *	OrigLoopID
	)

protected

Complete the loop skeleton by adding debug MDs, creating appropriate conditional branches in the middle block, preparing the builder and running the verifier.

Take in the vector loop L as argument, and return the preheader of the completed vector loop.

Definition at line 3441 of file LoopVectorize.cpp.

References assert(), Builder, Cost, llvm::CmpInst::Create(), DT, llvm::CallingConv::Fast, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::BasicBlock::getFirstInsertionPt(), llvm::Loop::getLoopID(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), getOrCreateTripCount(), getOrCreateVectorTripCount(), llvm::BasicBlock::getTerminator(), llvm::Optional< T >::getValue(), llvm::Optional< T >::hasValue(), llvm::CmpInst::ICMP_EQ, LI, LLVMLoopVectorizeFollowupAll, LLVMLoopVectorizeFollowupVectorized, LoopMiddleBlock, LoopVectorBody, LoopVectorPreHeader, llvm::makeFollowupLoopID(), ORE, OrigLoop, llvm::LoopVectorizeHints::setAlreadyVectorized(), llvm::Instruction::setDebugLoc(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Loop::setLoopID(), VectorTripCount, llvm::DominatorTreeBase< NodeT, IsPostDom >::verify(), and llvm::LoopInfoBase< BlockT, LoopT >::verify().

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ createBitOrPointerCast()

Value * InnerLoopVectorizer::createBitOrPointerCast	(	Value *	V,
		VectorType *	DstVTy,
		const DataLayout &	DL
	)

protected

Returns a bitcasted value to the requested vector type.

Also handles bitcasts of vector<float> <-> vector<pointer> types.

Definition at line 3043 of file LoopVectorize.cpp.

References assert(), Builder, llvm::IRBuilderBase::CreateBitOrPointerCast(), DL, llvm::FixedVectorType::get(), llvm::Value::getContext(), llvm::Type::getIntNTy(), llvm::Value::getType(), llvm::CastInst::isBitOrNoopPointerCastable(), llvm::Type::isFloatingPointTy(), llvm::Type::isPointerTy(), and VF.

Referenced by vectorizeInterleaveGroup().

◆ createInductionResumeValues()

void InnerLoopVectorizer::createInductionResumeValues	(	Loop *	L,
		Value *	VectorTripCount,
		std::pair< BasicBlock , Value >	AdditionalBypass = `{nullptr, nullptr}`
	)

protected

Create new phi nodes for the induction variables to resume iteration count in the scalar epilogue, from where the vectorized loop left off (given by VectorTripCount).

In cases where the loop skeleton is more complicated (eg. epilogue vectorization) and the resume values can come from an additional bypass block, the AdditionalBypass pair provides information about the bypass block and the end value on the edge from bypass to this loop.

Definition at line 3372 of file LoopVectorize.cpp.

References llvm::PHINode::addIncoming(), assert(), B, llvm::PHINode::Create(), DL, emitTransformedIndex(), llvm::CastInst::getCastOpcode(), llvm::Module::getDataLayout(), llvm::Instruction::getDebugLoc(), llvm::LoopVectorizationLegality::getInductionVars(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BasicBlock::getModule(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::getStartValue(), llvm::InductionDescriptor::getStep(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), IVEndValues, Legal, LoopBypassBlocks, LoopMiddleBlock, LoopScalarBody, LoopScalarPreHeader, OldInduction, PSE, llvm::Instruction::setDebugLoc(), llvm::PHINode::setIncomingValueForBlock(), llvm::Value::setName(), and VectorTripCount.

Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ createInductionVariable()

PHINode * InnerLoopVectorizer::createInductionVariable	(	Loop *	L,
		Value *	Start,
		Value *	End,
		Value *	Step,
		Instruction *	DL
	)

protected

Create a new induction variable inside L.

Definition at line 2910 of file LoopVectorize.cpp.

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ createVectorIntOrFpInductionPHI()

void InnerLoopVectorizer::createVectorIntOrFpInductionPHI	(	const InductionDescriptor &	II,
		Value *	Step,
		Value *	Start,
		Instruction *	EntryVal
	)

protected

Create a vector induction phi node based on an existing scalar one.

EntryVal is the value from the original loop that maps to the vector phi node, and Step is the loop-invariant step. If EntryVal is a truncate instruction, instead of widening the original IV, we widen a version of the IV truncated to EntryVal's type.

Definition at line 2019 of file LoopVectorize.cpp.

◆ createVectorizedLoopSkeleton()

BasicBlock * InnerLoopVectorizer::createVectorizedLoopSkeleton ( )

virtual

Create a new empty loop that will contain vectorized instructions later on, while the old loop will be used as the scalar remainder.

Control flow is generated around the vectorized (and scalar epilogue) loops consisting of various checks and bypasses. Return the pre-header block of the new loop. In the case of epilogue vectorization, this function is overriden to handle the more complex control flow around the loops.

Reimplemented in llvm::InnerLoopAndEpilogueVectorizer.

Definition at line 3500 of file LoopVectorize.cpp.

References Builder, completeLoopSkeleton(), createInductionResumeValues(), createInductionVariable(), createStepForVF(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), emitSCEVChecks(), llvm::ConstantInt::get(), getDebugLocFromInstOrOperands(), llvm::BasicBlock::getFirstInsertionPt(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getLoopID(), getOrCreateVectorTripCount(), llvm::LoopVectorizationLegality::getPrimaryInduction(), llvm::LoopVectorizationLegality::getWidestInductionType(), Induction, Legal, LoopScalarPreHeader, OldInduction, OrigLoop, llvm::IRBuilderBase::SetInsertPoint(), UF, and VF.

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ createVectorLoopSkeleton()

Loop * InnerLoopVectorizer::createVectorLoopSkeleton ( StringRef Prefix )

protected

Emit basic blocks (prefixed with Prefix) for the iteration check, vector loop preheader, middle block and scalar preheader.

Also allocate a loop object for the new vector loop and return it.

Definition at line 3324 of file LoopVectorize.cpp.

References llvm::LoopBase< BlockT, LoopT >::addBasicBlockToLoop(), llvm::LoopBase< BlockT, LoopT >::addChildLoop(), llvm::LoopInfoBase< BlockT, LoopT >::addTopLevelLoop(), llvm::LoopInfoBase< BlockT, LoopT >::AllocateLoop(), assert(), Builder, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), llvm::BranchInst::Create(), DT, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::LoopBase< BlockT, LoopT >::getParentLoop(), llvm::BasicBlock::getTerminator(), llvm::IRBuilderBase::getTrue(), llvm::LoopBase< BlockT, LoopT >::getUniqueExitBlock(), LI, LoopExitBlock, LoopMiddleBlock, LoopScalarBody, LoopScalarPreHeader, LoopVectorBody, LoopVectorPreHeader, OrigLoop, llvm::cl::Prefix, llvm::ReplaceInstWithInst(), llvm::Instruction::setDebugLoc(), and llvm::SplitBlock().

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ emitMemRuntimeChecks()

void InnerLoopVectorizer::emitMemRuntimeChecks	(	Loop *	L,
		BasicBlock *	Bypass
	)

protected

Emit bypass checks to check any memory assumptions we may have made.

Definition at line 3157 of file LoopVectorize.cpp.

References AddedSafetyChecks, llvm::addRuntimeChecks(), assert(), Builder, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), Cost, llvm::IRBuilderBase::CreateCondBr(), DEBUG_TYPE, DT, llvm::OptimizationRemarkEmitter::emit(), EnableVPlanNativePath, llvm::LoopVectorizeHints::FK_Enabled, llvm::RuntimePointerChecking::getChecks(), llvm::LoopVectorizeHints::getForce(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::getLAI(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BasicBlock::getParent(), llvm::LoopAccessInfo::getRuntimePointerChecking(), llvm::PredicatedScalarEvolution::getSE(), llvm::Loop::getStartLoc(), llvm::BasicBlock::getTerminator(), llvm::IRBuilderBase::getTrue(), llvm::Function::hasOptSize(), llvm::LoopVectorizationCostModel::Hints, Legal, LI, LoopBypassBlocks, LoopExitBlock, LoopVectorPreHeader, LVer, OptForSizeBasedOnProfile, ORE, OrigLoop, PSE, llvm::ReplaceInstWithInst(), llvm::Value::setName(), and llvm::SplitBlock().

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ emitMinimumIterationCountCheck()

void InnerLoopVectorizer::emitMinimumIterationCountCheck	(	Loop *	L,
		BasicBlock *	Bypass
	)

protected

Emit a bypass check to see if the vector trip count is zero, including if it overflows.

Definition at line 3074 of file LoopVectorize.cpp.

References assert(), Builder, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), Cost, llvm::BranchInst::Create(), llvm::IRBuilderBase::CreateICmp(), createStepForVF(), DT, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::ConstantInt::get(), llvm::IRBuilderBase::getFalse(), llvm::DomTreeNodeBase< NodeT >::getIDom(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), getOrCreateTripCount(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, LI, LoopBypassBlocks, LoopExitBlock, LoopVectorPreHeader, P, llvm::DominatorTreeBase< NodeT, IsPostDom >::properlyDominates(), llvm::ReplaceInstWithInst(), llvm::LoopVectorizationCostModel::requiresScalarEpilogue(), llvm::SplitBlock(), UF, and VF.

◆ emitSCEVChecks()

void InnerLoopVectorizer::emitSCEVChecks	(	Loop *	L,
		BasicBlock *	Bypass
	)

protected

Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct.

Definition at line 3116 of file LoopVectorize.cpp.

References AddedSafetyChecks, assert(), C, llvm::DominatorTreeBase< NodeT, IsPostDom >::changeImmediateDominator(), Cost, llvm::BranchInst::Create(), DT, llvm::LoopVectorizeHints::FK_Enabled, llvm::Module::getDataLayout(), llvm::LoopVectorizeHints::getForce(), llvm::BasicBlock::getModule(), llvm::BasicBlock::getParent(), llvm::PredicatedScalarEvolution::getSE(), llvm::BasicBlock::getTerminator(), llvm::PredicatedScalarEvolution::getUnionPredicate(), llvm::Function::hasOptSize(), llvm::LoopVectorizationCostModel::Hints, LI, LoopBypassBlocks, LoopExitBlock, LoopVectorPreHeader, OptForSizeBasedOnProfile, PSE, llvm::ReplaceInstWithInst(), llvm::Value::setName(), and llvm::SplitBlock().

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ emitTransformedIndex()

Value * InnerLoopVectorizer::emitTransformedIndex	(	IRBuilder<> &	B,
		Value *	Index,
		ScalarEvolution *	SE,
		const DataLayout &	DL,
		const InductionDescriptor &	ID
	)		const

protected

Compute the transformed value of Index at offset StartValue using step StepValue.

For integer induction, returns StartValue + Index * StepValue. For pointer induction, returns StartValue[Index * StepValue]. FIXME: The newly created binary instructions should contain nsw/nuw flags, which can be found from the original scalar operations.

Definition at line 3224 of file LoopVectorize.cpp.

References assert(), B, CreateAdd(), CreateMul(), DL, llvm::FAdd, llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::BasicBlock::getTerminator(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, Index, LI, llvm_unreachable, LoopVectorBody, Offset, llvm::FastMathFlags::setFast(), X, and Y.

Referenced by createInductionResumeValues(), fixupIVUsers(), and widenPHIInstruction().

◆ fixCrossIterationPHIs()

void InnerLoopVectorizer::fixCrossIterationPHIs ( )

protected

Handle all cross-iteration phis in the header.

Definition at line 3962 of file LoopVectorize.cpp.

References fixFirstOrderRecurrence(), fixReduction(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::isFirstOrderRecurrence(), llvm::LoopVectorizationLegality::isReductionVariable(), Legal, OrigLoop, and llvm::BasicBlock::phis().

Referenced by fixVectorizedLoop().

◆ fixFirstOrderRecurrence()

void InnerLoopVectorizer::fixFirstOrderRecurrence ( PHINode * Phi )

protected

Fix a first-order recurrence.

This is the second phase of vectorizing this phi node.

Definition at line 3978 of file LoopVectorize.cpp.

References llvm::PHINode::addIncoming(), assert(), llvm::BasicBlock::begin(), Builder, llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::VectorType::get(), llvm::PoisonValue::get(), llvm::BasicBlock::getFirstInsertionPt(), llvm::PHINode::getIncomingValueForBlock(), llvm::IRBuilderBase::getInt32(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), getOrCreateVectorValue(), llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), llvm::VectorizerValueMap::getVectorValue(), I, llvm::Loop::isLoopInvariant(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), LI, LoopMiddleBlock, LoopScalarPreHeader, LoopVectorBody, LoopVectorPreHeader, OrigLoop, llvm::Value::replaceAllUsesWith(), llvm::VectorizerValueMap::resetVectorValue(), llvm::IRBuilderBase::SetInsertPoint(), UF, VectorLoopValueMap, and VF.

Referenced by fixCrossIterationPHIs().

◆ fixLCSSAPHIs()

void InnerLoopVectorizer::fixLCSSAPHIs ( )

protected

Fixup the LCSSA phi nodes in the unique exit block.

This simply means we need to add the appropriate incoming value from the middle block as exiting edges from the scalar epilogue loop (if present) are already in place, and we exit the vector loop exclusively to the middle block.

Definition at line 4368 of file LoopVectorize.cpp.

References assert(), Builder, Cost, llvm::LinearPolySize< LeafTy >::getKnownMinValue(), getOrCreateScalarValue(), llvm::BasicBlock::getTerminator(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), LoopExitBlock, LoopMiddleBlock, llvm::BasicBlock::phis(), llvm::IRBuilderBase::SetInsertPoint(), UF, and VF.

◆ fixNonInductionPHIs()

void InnerLoopVectorizer::fixNonInductionPHIs ( void )

Fix the non-induction PHIs in the OrigPHIsToFix vector.

Definition at line 4458 of file LoopVectorize.cpp.

References llvm::PHINode::addIncoming(), assert(), Builder, llvm::PHINode::getNumIncomingValues(), getOrCreateVectorValue(), llvm::Instruction::getParent(), llvm::VectorizerValueMap::getVectorValue(), OrigPHIsToFix, llvm::predecessors(), llvm::IRBuilderBase::SetInsertPoint(), llvm::SmallVectorBase< SmallVectorSizeType< T > >::size(), and VectorLoopValueMap.

Referenced by fixVectorizedLoop().

◆ fixReduction()

void InnerLoopVectorizer::fixReduction ( PHINode * Phi )

protected

Fix a reduction cross-iteration phi.

This is the second phase of vectorizing this phi node.

Definition at line 4163 of file LoopVectorize.cpp.

References addFastMathFlag(), llvm::PHINode::addIncoming(), llvm::any_of(), assert(), Builder, clearReductionWrapFlags(), Cost, llvm::PHINode::Create(), llvm::IRBuilderBase::CreateBinOp(), llvm::createMinMaxOp(), llvm::IRBuilderBase::CreateSExt(), llvm::createTargetReduction(), llvm::IRBuilderBase::CreateTrunc(), llvm::IRBuilderBase::CreateZExt(), E, llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::VectorType::get(), llvm::PHINode::getBasicBlockIndex(), llvm::BasicBlock::getFirstInsertionPt(), llvm::PHINode::getIncomingValueForBlock(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::RecurrenceDescriptor::getOpcode(), getOrCreateVectorValue(), llvm::LoopVectorizationLegality::getReductionVars(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::VectorizerValueMap::getVectorValue(), I, llvm::LoopVectorizationCostModel::isInLoopReduction(), llvm::LoopVectorizationLegality::isReductionVariable(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), Legal, LI, LoopBypassBlocks, LoopExitBlock, LoopMiddleBlock, LoopScalarPreHeader, LoopVectorBody, OrigLoop, llvm::BasicBlock::phis(), PreferPredicatedReductionSelect, llvm::TargetTransformInfo::preferPredicatedReductionSelect(), replaceUsesOfWith(), llvm::VectorizerValueMap::resetVectorValue(), setDebugLocFromInst(), llvm::PHINode::setIncomingValue(), llvm::IRBuilderBase::SetInsertPoint(), UF, llvm::Value::users(), VectorLoopValueMap, and VF.

Referenced by fixCrossIterationPHIs().

◆ fixupIVUsers()

void InnerLoopVectorizer::fixupIVUsers	(	PHINode *	OrigPhi,
		const InductionDescriptor &	II,
		Value *	CountRoundDown,
		Value *	EndValue,
		BasicBlock *	MiddleBlock
	)

protected

Set up the values of the IVs correctly when exiting the vector loop.

Definition at line 3589 of file LoopVectorize.cpp.

References llvm::PHINode::addIncoming(), assert(), B, llvm::LoopBase< BlockT, LoopT >::contains(), DL, emitTransformedIndex(), llvm::ConstantInt::get(), llvm::PHINode::getBasicBlockIndex(), llvm::Module::getDataLayout(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::PHINode::getIncomingValueForBlock(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::BasicBlock::getModule(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::getStep(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::LoopBase< BlockT, LoopT >::getUniqueExitBlock(), I, llvm::Type::isIntegerTy(), OrigLoop, PostInc, PSE, llvm::Value::setName(), and llvm::Value::users().

◆ fixVectorizedLoop()

void InnerLoopVectorizer::fixVectorizedLoop ( )

Fix the vectorized code, taking care of header phi's, live-outs, and more.

Definition at line 3909 of file LoopVectorize.cpp.

References assert(), EnableVPlanNativePath, fixCrossIterationPHIs(), fixNonInductionPHIs(), llvm::ScalarEvolution::forgetLoop(), llvm::PredicatedScalarEvolution::getSE(), llvm::ElementCount::isVector(), OrigLoop, OrigPHIsToFix, PSE, truncateToMinimalBitwidths(), and VF.

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ getBroadcastInstrs()

Value * InnerLoopVectorizer::getBroadcastInstrs ( Value * V )

protectedvirtual

Create a broadcast instruction.

This method generates a broadcast instruction (shuffle) for loop invariant values and for the induction value. If this is the induction variable then we extend it to N, N+1, ... this is needed because each iteration in the loop corresponds to a SIMD element.

Definition at line 2000 of file LoopVectorize.cpp.

References Builder, llvm::IRBuilderBase::CreateVectorSplat(), llvm::DominatorTree::dominates(), DT, llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), llvm::Loop::isLoopInvariant(), LoopVectorPreHeader, OrigLoop, llvm::IRBuilderBase::SetInsertPoint(), and VF.

Referenced by getOrCreateVectorValue().

◆ getOrCreateScalarValue()

Value * InnerLoopVectorizer::getOrCreateScalarValue	(	Value *	V,
		const VPIteration &	Instance
	)

Return a value in the new loop corresponding to V from the original loop at unroll and vector indices Instance.

If the value has been vectorized but not scalarized, the necessary extractelement instruction will be generated.

Definition at line 2441 of file LoopVectorize.cpp.

References assert(), Builder, Cost, llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::getInt32(), getOrCreateVectorValue(), llvm::VectorizerValueMap::getScalarValue(), llvm::VectorizerValueMap::hasScalarValue(), llvm::Loop::isLoopInvariant(), llvm::ElementCount::isScalar(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::VPIteration::Lane, OrigLoop, llvm::VPIteration::Part, VectorLoopValueMap, and VF.

Referenced by fixLCSSAPHIs().

◆ getOrCreateTripCount()

Value * InnerLoopVectorizer::getOrCreateTripCount ( Loop * NewLoop )

protected

Returns (and creates if needed) the original loop trip count.

Definition at line 2942 of file LoopVectorize.cpp.

References assert(), Builder, llvm::CastInst::CreatePointerCast(), DL, llvm::PredicatedScalarEvolution::getBackedgeTakenCount(), llvm::Module::getDataLayout(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BasicBlock::getModule(), llvm::Type::getPrimitiveSizeInBits(), llvm::PredicatedScalarEvolution::getSE(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::LoopVectorizationLegality::getWidestInductionType(), llvm::Type::isPointerTy(), Legal, PSE, and TripCount.

Referenced by completeLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::LoopVectorizationPlanner::executePlan(), and getOrCreateVectorTripCount().

◆ getOrCreateVectorTripCount()

Value * InnerLoopVectorizer::getOrCreateVectorTripCount ( Loop * NewLoop )

protected

Returns (and creates if needed) the trip count of the widened loop.

Definition at line 2989 of file LoopVectorize.cpp.

References assert(), Builder, Cost, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateSelect(), createStepForVF(), llvm::IRBuilderBase::CreateSub(), llvm::IRBuilderBase::CreateURem(), llvm::LoopVectorizationCostModel::foldTailByMasking(), llvm::ConstantInt::get(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), getOrCreateTripCount(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::isPowerOf2_32(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), llvm::LoopVectorizationCostModel::requiresScalarEpilogue(), UF, VectorTripCount, and VF.

Referenced by completeLoopSkeleton(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), and createVectorizedLoopSkeleton().

◆ getOrCreateVectorValue()

Value * InnerLoopVectorizer::getOrCreateVectorValue	(	Value *	V,
		unsigned	Part
	)

getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a vector or scalar value on-demand if one is not yet available.

When vectorizing a loop, we visit the definition of an instruction before its uses. When visiting the definition, we either vectorize or scalarize the instruction, creating an entry for it in the corresponding map. (In some cases, such as induction variables, we will create both vector and scalar entries.) Then, as we encounter uses of the definition, we derive values for each scalar or vector use unless such a value is already available. For example, if we scalarize a definition and one of its uses is vector, we build the required vector on-demand with an insertelement sequence when visiting the use. Otherwise, if the use is scalar, we can use the existing scalar definition.

Return a value in the new loop corresponding to V from the original loop at unroll index Part. If the value has already been vectorized, the corresponding vector entry in VectorLoopValueMap is returned. If, however, the value has a scalar entry in VectorLoopValueMap, we construct a new vector value on-demand by inserting the scalar values into a vector with an insertelement sequence. If the value has been neither vectorized nor scalarized, it must be loop invariant, so we simply broadcast the value into a vector.

Definition at line 2362 of file LoopVectorize.cpp.

References assert(), B, Builder, Cost, EnableVPlanNativePath, llvm::ConstantInt::get(), llvm::VectorType::get(), llvm::PoisonValue::get(), getBroadcastInstrs(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::VectorizerValueMap::getScalarValue(), llvm::Value::getType(), llvm::VectorizerValueMap::getVectorValue(), llvm::VectorizerValueMap::hasAnyScalarValue(), llvm::LoopVectorizationLegality::hasStride(), llvm::VectorizerValueMap::hasVectorValue(), I, Induction, llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isScalar(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::Type::isVectorTy(), llvm::Type::isVoidTy(), Legal, packScalarIntoVectorValue(), llvm::IRBuilderBase::restoreIP(), llvm::IRBuilderBase::saveIP(), llvm::IRBuilderBase::SetInsertPoint(), llvm::VectorizerValueMap::setVectorValue(), VectorLoopValueMap, and VF.

Referenced by clearReductionWrapFlags(), fixFirstOrderRecurrence(), fixNonInductionPHIs(), fixReduction(), getOrCreateScalarValue(), and truncateToMinimalBitwidths().

◆ getStepVector()

Value * InnerLoopVectorizer::getStepVector	(	Value *	Val,
		int	StartIdx,
		Value *	Step,
		Instruction::BinaryOps	Opcode = `Instruction::BinaryOpsEnd`
	)

protectedvirtual

This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...) to each vector element of Val.

The sequence starts at StartIndex. Opcode is relevant for FP induction variable.

Definition at line 2251 of file LoopVectorize.cpp.

References assert(), Builder, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateFMul(), llvm::IRBuilderBase::CreateMul(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::FAdd, llvm::ConstantInt::get(), llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::FastMathFlags::setFast().

◆ needsScalarInduction()

bool InnerLoopVectorizer::needsScalarInduction ( Instruction * IV ) const

protected

Returns true if we should generate a scalar version of IV.

Definition at line 2103 of file LoopVectorize.cpp.

References llvm::any_of(), llvm::LoopBase< BlockT, LoopT >::contains(), I, OrigLoop, shouldScalarizeInstruction(), and llvm::Value::users().

◆ packScalarIntoVectorValue()

void InnerLoopVectorizer::packScalarIntoVectorValue	(	Value *	V,
		const VPIteration &	Instance
	)

Construct the vector value of a scalarized value V one lane at a time.

Definition at line 2474 of file LoopVectorize.cpp.

References assert(), Builder, llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::getInt32(), llvm::VectorizerValueMap::getScalarValue(), llvm::Value::getType(), llvm::VectorizerValueMap::getVectorValue(), Induction, llvm::Type::isVectorTy(), llvm::Type::isVoidTy(), llvm::VPIteration::Lane, llvm::VPIteration::Part, llvm::VectorizerValueMap::resetVectorValue(), and VectorLoopValueMap.

Referenced by llvm::VPReplicateRecipe::execute(), and getOrCreateVectorValue().

◆ printDebugTracesAtEnd()

virtual void llvm::InnerLoopVectorizer::printDebugTracesAtEnd ( )

inlineprotectedvirtual

Reimplemented in llvm::EpilogueVectorizerEpilogueLoop, and llvm::EpilogueVectorizerMainLoop.

Definition at line 791 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ printDebugTracesAtStart()

virtual void llvm::InnerLoopVectorizer::printDebugTracesAtStart ( )

inlineprotectedvirtual

Allow subclasses to override and print debug traces before/after vplan execution, when trace information is requested.

Reimplemented in llvm::EpilogueVectorizerEpilogueLoop, and llvm::EpilogueVectorizerMainLoop.

Definition at line 790 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ recordVectorLoopValueForInductionCast()

void InnerLoopVectorizer::recordVectorLoopValueForInductionCast	(	const InductionDescriptor &	ID,
		const Instruction *	EntryVal,
		Value *	VectorLoopValue,
		unsigned	Part,
		unsigned	Lane = `UINT_MAX`
	)

protected

If there is a cast involved in the induction variable ID, which should be ignored in the vectorized loop body, this function records the VectorLoopValue of the respective Phi also as the VectorLoopValue of the cast.

We had already proved that the casted Phi is equal to the uncasted Phi in the vectorized loop (under a runtime guard), and therefore there is no need to vectorize the cast - the same value can be used in the vector loop for both the Phi and the cast. If VectorLoopValue is a scalarized value, Lane is also specified, Otherwise, VectorLoopValue is a widened/vectorized value.

EntryVal is the value from the original loop that maps to the vector phi node and is used to distinguish what is the IV currently being processed - original one (if EntryVal is a phi corresponding to the original IV) or the "newly-created" one based on the proof mentioned above (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the latter case EntryVal is a TruncInst and we must not record anything for that IV, but it's error-prone to expect callers of this routine to care about that, hence this explicit parameter.

Definition at line 2113 of file LoopVectorize.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorBase< SmallVectorSizeType< T > >::empty(), llvm::VectorizerValueMap::setScalarValue(), llvm::VectorizerValueMap::setVectorValue(), and VectorLoopValueMap.

Referenced by buildScalarSteps().

◆ reverseVector()

Value * InnerLoopVectorizer::reverseVector ( Value * Vec )

protectedvirtual

Generate a shuffle sequence that will reverse the vector Vec.

Definition at line 2487 of file LoopVectorize.cpp.

References assert(), Builder, llvm::IRBuilderBase::CreateShuffleVector(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::Value::getType(), llvm::LinearPolySize< LeafTy >::isScalable(), llvm::Type::isVectorTy(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and VF.

Referenced by vectorizeInterleaveGroup(), and vectorizeMemoryInstruction().

◆ scalarizeInstruction()

void InnerLoopVectorizer::scalarizeInstruction	(	Instruction *	Instr,
		VPUser &	Operands,
		const VPIteration &	Instance,
		bool	IfPredicateInstr,
		VPTransformState &	State
	)

A helper function to scalarize a single Instruction in the innermost loop.

Generates a sequence of scalar instances for each lane between MinLane and MaxLane, times each part between MinPart and MaxPart, inclusive. Uses the VPValue operands from Operands instead of Instr's operands.

Definition at line 2864 of file LoopVectorize.cpp.

References AC, addNewMetadata(), assert(), Builder, llvm::Instruction::clone(), llvm::LoopBase< BlockT, LoopT >::contains(), Cost, llvm::numbers::e, llvm::VPTransformState::get(), llvm::Value::getName(), llvm::User::getNumOperands(), llvm::User::getOperand(), llvm::Value::getType(), llvm::IRBuilderBase::Insert(), llvm::Type::isAggregateType(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::Type::isVoidTy(), llvm::VPIteration::Lane, op, OrigLoop, PredicatedInstructions, llvm::AssumptionCache::registerAssumption(), setDebugLocFromInst(), llvm::Value::setName(), llvm::User::setOperand(), llvm::VectorizerValueMap::setScalarValue(), VectorLoopValueMap, and llvm::VPTransformState::VF.

Referenced by llvm::VPReplicateRecipe::execute().

◆ setDebugLocFromInst()

void InnerLoopVectorizer::setDebugLocFromInst	(	IRBuilder<> &	B,
		const Value *	Ptr
	)

Set the debug location in the builder using the debug location in the instruction.

Definition at line 1076 of file LoopVectorize.cpp.

References assert(), B, llvm::DILocation::cloneByMultiplyingDuplicationFactor(), llvm::dbgs(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LinearPolySize< LeafTy >::isScalable(), LLVM_DEBUG, UF, and VF.

Referenced by llvm::VPBlendRecipe::execute(), fixReduction(), scalarizeInstruction(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().

◆ setVectorValue()

void llvm::InnerLoopVectorizer::setVectorValue	(	Value *	Scalar,
		unsigned	Part,
		Value *	Vector
	)

inline

Definition at line 563 of file LoopVectorize.cpp.

References llvm::NVPTX::PTXLdStInstCode::Scalar, llvm::VectorizerValueMap::setVectorValue(), llvm::Vector, and VectorLoopValueMap.

◆ shouldScalarizeInstruction()

bool InnerLoopVectorizer::shouldScalarizeInstruction ( Instruction * I ) const

protected

Returns true if an instruction I should be scalarized instead of vectorized for the chosen vectorization factor.

Definition at line 2098 of file LoopVectorize.cpp.

References Cost, I, llvm::LoopVectorizationCostModel::isProfitableToScalarize(), llvm::LoopVectorizationCostModel::isScalarAfterVectorization(), and VF.

Referenced by needsScalarInduction().

◆ sinkScalarOperands()

void InnerLoopVectorizer::sinkScalarOperands ( Instruction * PredInst )

protected

Iteratively sink the scalarized operands of a predicated instruction into the block that was created for it.

Definition at line 4394 of file LoopVectorize.cpp.

References llvm::all_of(), llvm::SetVector< T, Vector, Set >::begin(), llvm::SetVector< T, Vector, Set >::clear(), llvm::SetVector< T, Vector, Set >::end(), llvm::PHINode::getIncomingValueNumForOperand(), llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(), llvm::Instruction::getParent(), I, LI, llvm::User::op_begin(), and llvm::User::op_end().

◆ truncateToMinimalBitwidths()

void InnerLoopVectorizer::truncateToMinimalBitwidths ( )

protected

Shrinks vector element sizes to the smallest bitwidth they can be legally represented as.

Definition at line 3775 of file LoopVectorize.cpp.

References B, Cost, llvm::SmallPtrSetImpl< PtrType >::count(), llvm::Instruction::eraseFromParent(), llvm::IntegerType::get(), llvm::FixedVectorType::get(), llvm::Type::getContext(), llvm::LoopVectorizationCostModel::getMinimalBitwidths(), llvm::User::getOperand(), getOrCreateVectorValue(), llvm::VectorizerValueMap::hasAnyVectorValue(), I, llvm::ARM_PROC::IE, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm_unreachable, llvm::VectorizerValueMap::resetVectorValue(), SI, smallestIntegerVectorType(), llvm::Value::takeName(), UF, llvm::Value::use_empty(), and VectorLoopValueMap.

Referenced by fixVectorizedLoop().

◆ vectorizeInterleaveGroup()

void InnerLoopVectorizer::vectorizeInterleaveGroup	(	const InterleaveGroup< Instruction > *	Group,
		ArrayRef< VPValue * >	VPDefs,
		VPTransformState &	State,
		VPValue *	Addr,
		ArrayRef< VPValue * >	StoredValues,
		VPValue *	BlockInMask = `nullptr`
	)

Try to vectorize interleaved access group Group with the base address given in Addr, optionally masking the vector operations if BlockInMask is non-null.

Use State to translate given VPValues to IR values in the vectorized loop.

Definition at line 2536 of file LoopVectorize.cpp.

Referenced by llvm::VPInterleaveRecipe::execute().

◆ vectorizeMemoryInstruction()

void InnerLoopVectorizer::vectorizeMemoryInstruction	(	Instruction *	Instr,
		VPTransformState &	State,
		VPValue *	Def,
		VPValue *	Addr,
		VPValue *	StoredValue,
		VPValue *	BlockInMask
	)

Vectorize Load and Store instructions with the base address given in Addr, optionally masking the vector operations if BlockInMask is non-null.

Use State to translate given VPValues to IR values in the vectorized loop.

Definition at line 2725 of file LoopVectorize.cpp.

Referenced by llvm::VPWidenMemoryInstructionRecipe::execute().

◆ widenCallInstruction()

void InnerLoopVectorizer::widenCallInstruction	(	CallInst &	I,
		VPValue *	Def,
		VPUser &	ArgOperands,
		VPTransformState &	State
	)

Widen a single call instruction within the innermost loop.

Definition at line 4863 of file LoopVectorize.cpp.

References addMetadata(), Arg, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), Builder, llvm::Instruction::copyFastMathFlags(), Cost, llvm::IRBuilderBase::CreateCall(), llvm::tgtok::Def, llvm::enumerate(), llvm::VFShape::get(), llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::LoopVectorizationCostModel::getVectorCallCost(), llvm::LoopVectorizationCostModel::getVectorIntrinsicCost(), llvm::getVectorIntrinsicIDForCall(), llvm::VFDatabase::getVectorizedFunction(), llvm::hasVectorInstrinsicScalarOpd(), I, llvm::LinearPolySize< LeafTy >::isScalable(), llvm::ElementCount::isVector(), llvm::VPUser::operands(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::VPTransformState::set(), setDebugLocFromInst(), TLI, llvm::ToVectorTy(), UF, and VF.

Referenced by llvm::VPWidenCallRecipe::execute().

◆ widenGEP()

void InnerLoopVectorizer::widenGEP	(	GetElementPtrInst *	GEP,
		VPValue *	VPDef,
		VPUser &	Indices,
		unsigned	UF,
		ElementCount	VF,
		bool	IsPtrLoopInvariant,
		SmallBitVector &	IsIndexLoopInvariant,
		VPTransformState &	State
	)

Vectorize a single GetElementPtrInst based on information gathered and decisions taken during planning.

Definition at line 4494 of file LoopVectorize.cpp.

References addMetadata(), llvm::SmallBitVector::all(), assert(), Builder, llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreateVectorSplat(), E, GEP, llvm::VPTransformState::get(), I, llvm::IRBuilderBase::Insert(), llvm::ElementCount::isScalar(), llvm::ElementCount::isVector(), Operands, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::VPTransformState::set(), UF, and VF.

Referenced by llvm::VPWidenGEPRecipe::execute().

◆ widenInstruction()

void InnerLoopVectorizer::widenInstruction	(	Instruction &	I,
		VPValue *	Def,
		VPUser &	Operands,
		VPTransformState &	State
	)

Widen a single instruction within the innermost loop.

Vectorize casts.

Definition at line 4755 of file LoopVectorize.cpp.

References llvm::Add, addMetadata(), llvm::And, B, Builder, C, llvm::MCID::Call, llvm::IRBuilderBase::CreateCast(), llvm::IRBuilderBase::CreateFCmp(), llvm::IRBuilderBase::CreateICmp(), llvm::IRBuilderBase::CreateNAryOp(), llvm::dbgs(), llvm::tgtok::Def, llvm::FAdd, llvm::FMul, llvm::VPTransformState::get(), llvm::VectorType::get(), llvm::User::getOperand(), I, llvm::ElementCount::isScalar(), LLVM_DEBUG, llvm_unreachable, llvm::Mul, llvm::User::operands(), llvm::Or, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MCID::Select, llvm::VPTransformState::set(), setDebugLocFromInst(), llvm::IRBuilderBase::setFastMathFlags(), UF, VF, and llvm::Xor.

Referenced by llvm::VPWidenRecipe::execute().

◆ widenIntOrFpInduction()

void InnerLoopVectorizer::widenIntOrFpInduction	(	PHINode *	IV,
		Value *	Start,
		TruncInst *	Trunc = `nullptr`
	)

Widen an integer or floating-point induction variable IV.

If Trunc is provided, the integer induction variable will first be truncated to the corresponding type.

Definition at line 2141 of file LoopVectorize.cpp.

Referenced by llvm::VPWidenIntOrFpInductionRecipe::execute().

◆ widenPHIInstruction()

void InnerLoopVectorizer::widenPHIInstruction	(	Instruction *	PN,
		RecurrenceDescriptor *	RdxDesc,
		Value *	StartV,
		unsigned	UF,
		ElementCount	VF
	)

Vectorize a single PHINode in a block.

This method handles the induction variable canonicalization. It supports both VF = 1 for unrolled loops and arbitrary length vectors.

Definition at line 4564 of file LoopVectorize.cpp.

Referenced by llvm::VPWidenPHIRecipe::execute().

◆ widenSelectInstruction()

void InnerLoopVectorizer::widenSelectInstruction	(	SelectInst &	I,
		VPValue *	VPDef,
		VPUser &	Operands,
		bool	InvariantCond,
		VPTransformState &	State
	)

Widen a single select instruction within the innermost loop.

Definition at line 4933 of file LoopVectorize.cpp.

References addMetadata(), Builder, Cond, llvm::IRBuilderBase::CreateSelect(), llvm::VPTransformState::get(), I, Operands, llvm::VPTransformState::set(), setDebugLocFromInst(), and UF.

Referenced by llvm::VPWidenSelectRecipe::execute().

Friends And Related Function Documentation

◆ LoopVectorizationPlanner

friend class LoopVectorizationPlanner

friend

Definition at line 602 of file LoopVectorize.cpp.

Member Data Documentation

◆ AA

AAResults* llvm::InnerLoopVectorizer::AA

protected

Alias Analysis.

Definition at line 808 of file LoopVectorize.cpp.

◆ AC

AssumptionCache* llvm::InnerLoopVectorizer::AC

protected

Assumption Cache.

Definition at line 817 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationCostModel::collectValuesToIgnore(), getScalarEpilogueLowering(), processLoopInVPlanNativePath(), and scalarizeInstruction().

◆ AddedSafetyChecks

bool llvm::InnerLoopVectorizer::AddedSafetyChecks = false

protected

Definition at line 892 of file LoopVectorize.cpp.

Referenced by areSafetyChecksAdded(), emitMemRuntimeChecks(), and emitSCEVChecks().

◆ BFI

BlockFrequencyInfo* llvm::InnerLoopVectorizer::BFI

protected

BFI and PSI are used to check for profile guided size optimizations.

Definition at line 903 of file LoopVectorize.cpp.

Referenced by getScalarEpilogueLowering(), InnerLoopVectorizer(), and processLoopInVPlanNativePath().

◆ Builder

IRBuilder llvm::InnerLoopVectorizer::Builder

protected

The builder that we use.

Definition at line 838 of file LoopVectorize.cpp.

Referenced by buildScalarSteps(), completeLoopSkeleton(), createBitOrPointerCast(), llvm::VPRecipeBuilder::createBlockInMask(), llvm::VPRecipeBuilder::createEdgeMask(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), llvm::LoopVectorizationPlanner::executePlan(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixNonInductionPHIs(), fixReduction(), getBroadcastInstrs(), getOrCreateScalarValue(), getOrCreateTripCount(), getOrCreateVectorTripCount(), getOrCreateVectorValue(), getStepVector(), packScalarIntoVectorValue(), reverseVector(), scalarizeInstruction(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().

◆ Cost

LoopVectorizationCostModel* llvm::InnerLoopVectorizer::Cost

protected

The profitablity analysis.

Definition at line 889 of file LoopVectorize.cpp.

Referenced by buildScalarSteps(), completeLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), fixLCSSAPHIs(), fixReduction(), getOrCreateScalarValue(), getOrCreateVectorTripCount(), getOrCreateVectorValue(), llvm::LoopVectorizationCostModel::getVectorCallCost(), scalarizeInstruction(), llvm::LoopVectorizationCostModel::selectVectorizationFactor(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::LoopVectorizationCostModel::setWideningDecision(), shouldScalarizeInstruction(), truncateToMinimalBitwidths(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), and widenPHIInstruction().

◆ DT

DominatorTree* llvm::InnerLoopVectorizer::DT

protected

Dominator Tree.

Definition at line 805 of file LoopVectorize.cpp.

Referenced by completeLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), llvm::LoopVectorizationPlanner::executePlan(), getBroadcastInstrs(), getScalarEpilogueLowering(), and processLoopInVPlanNativePath().

◆ Induction

PHINode* llvm::InnerLoopVectorizer::Induction = nullptr

protected

The new Induction variable which was added to the new block.

Definition at line 865 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), llvm::LoopVectorizationCostModel::collectValuesToIgnore(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), llvm::LoopVectorizationPlanner::executePlan(), getOrCreateVectorValue(), packScalarIntoVectorValue(), llvm::EpilogueVectorizerMainLoop::printDebugTracesAtEnd(), llvm::EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd(), and widenPHIInstruction().

◆ IVEndValues

DenseMap<PHINode *, Value *> llvm::InnerLoopVectorizer::IVEndValues

protected

Definition at line 896 of file LoopVectorize.cpp.

Referenced by createInductionResumeValues().

◆ Legal

LoopVectorizationLegality* llvm::InnerLoopVectorizer::Legal

protected

The legality analysis.

Definition at line 886 of file LoopVectorize.cpp.

◆ LI

LoopInfo* llvm::InnerLoopVectorizer::LI

protected

◆ LoopBypassBlocks

SmallVector<BasicBlock *, 4> llvm::InnerLoopVectorizer::LoopBypassBlocks

protected

A list of all bypass blocks. The first block is the entry of the loop.

Definition at line 862 of file LoopVectorize.cpp.

Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), and fixReduction().

◆ LoopExitBlock

BasicBlock* llvm::InnerLoopVectorizer::LoopExitBlock

protected

The (unique) ExitBlock of the scalar loop.

Note that there can be multiple exiting edges reaching this block.

Definition at line 853 of file LoopVectorize.cpp.

Referenced by llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), emitSCEVChecks(), fixLCSSAPHIs(), and fixReduction().

◆ LoopMiddleBlock

BasicBlock* llvm::InnerLoopVectorizer::LoopMiddleBlock

protected

Middle Block between the vector and the scalar.

Definition at line 849 of file LoopVectorize.cpp.

Referenced by completeLoopSkeleton(), createInductionResumeValues(), createVectorLoopSkeleton(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), and fixReduction().

◆ LoopScalarBody

BasicBlock* llvm::InnerLoopVectorizer::LoopScalarBody

protected

The scalar loop body.

Definition at line 859 of file LoopVectorize.cpp.

Referenced by createInductionResumeValues(), and createVectorLoopSkeleton().

◆ LoopScalarPreHeader

BasicBlock* llvm::InnerLoopVectorizer::LoopScalarPreHeader

protected

The scalar-loop preheader.

Definition at line 846 of file LoopVectorize.cpp.

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), fixFirstOrderRecurrence(), and fixReduction().

◆ LoopVectorBody

BasicBlock* llvm::InnerLoopVectorizer::LoopVectorBody

protected

The vector loop body.

Definition at line 856 of file LoopVectorize.cpp.

Referenced by completeLoopSkeleton(), createVectorLoopSkeleton(), emitTransformedIndex(), fixFirstOrderRecurrence(), fixReduction(), and widenPHIInstruction().

◆ LoopVectorPreHeader

BasicBlock* llvm::InnerLoopVectorizer::LoopVectorPreHeader

protected

The vector-loop preheader.

Definition at line 843 of file LoopVectorize.cpp.

Referenced by completeLoopSkeleton(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), emitSCEVChecks(), fixFirstOrderRecurrence(), getBroadcastInstrs(), and widenPHIInstruction().

◆ LVer

std::unique_ptr<LoopVersioning> llvm::InnerLoopVectorizer::LVer

protected

LoopVersioning.

It's only set up (non-null) if memchecks were used.

This is currently only used to add no-alias metadata based on the memchecks. The actually versioning is performed manually.

Definition at line 827 of file LoopVectorize.cpp.

Referenced by addNewMetadata(), and emitMemRuntimeChecks().

◆ OldInduction

PHINode* llvm::InnerLoopVectorizer::OldInduction = nullptr

protected

The induction variable of the old basic block.

Definition at line 868 of file LoopVectorize.cpp.

Referenced by llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createInductionResumeValues(), and createVectorizedLoopSkeleton().

◆ OptForSizeBasedOnProfile

bool llvm::InnerLoopVectorizer::OptForSizeBasedOnProfile

protected

Definition at line 908 of file LoopVectorize.cpp.

Referenced by emitMemRuntimeChecks(), emitSCEVChecks(), and InnerLoopVectorizer().

◆ ORE

OptimizationRemarkEmitter* llvm::InnerLoopVectorizer::ORE

protected

Interface to emit optimization remarks.

Definition at line 820 of file LoopVectorize.cpp.

Referenced by collectSupportedLoops(), completeLoopSkeleton(), llvm::LoopVectorizationCostModel::computeMaxVF(), emitMemRuntimeChecks(), isExplicitVecOuterLoop(), processLoopInVPlanNativePath(), llvm::reportVectorizationFailure(), llvm::LoopVectorizationCostModel::runtimeChecksRequired(), and llvm::LoopVectorizationCostModel::selectVectorizationFactor().

◆ OrigLoop

Loop* llvm::InnerLoopVectorizer::OrigLoop

protected

The original loop.

Definition at line 791 of file LoopVectorize.cpp.

Referenced by clearReductionWrapFlags(), llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), completeLoopSkeleton(), llvm::VPRecipeBuilder::createBlockInMask(), llvm::VPRecipeBuilder::createEdgeMask(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), createVectorLoopSkeleton(), emitMemRuntimeChecks(), fixCrossIterationPHIs(), fixFirstOrderRecurrence(), fixReduction(), fixupIVUsers(), fixVectorizedLoop(), getBroadcastInstrs(), getOrCreateScalarValue(), InnerLoopVectorizer(), needsScalarInduction(), llvm::LoopVectorizationPlanner::plan(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), scalarizeInstruction(), llvm::VPRecipeBuilder::tryToCreateWidenRecipe(), and widenPHIInstruction().

◆ OrigPHIsToFix

SmallVector<PHINode *, 8> llvm::InnerLoopVectorizer::OrigPHIsToFix

protected

Definition at line 900 of file LoopVectorize.cpp.

Referenced by fixNonInductionPHIs(), fixVectorizedLoop(), and widenPHIInstruction().

◆ PredicatedInstructions

SmallVector<Instruction *, 4> llvm::InnerLoopVectorizer::PredicatedInstructions

protected

Store instructions that were predicated.

Definition at line 877 of file LoopVectorize.cpp.

Referenced by scalarizeInstruction().

◆ PSE

PredicatedScalarEvolution& llvm::InnerLoopVectorizer::PSE

protected

A wrapper around ScalarEvolution used to add runtime SCEV checks.

Applies dynamic knowledge to simplify SCEV expressions and converts them to a more usable form.

Definition at line 799 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), createInductionResumeValues(), emitMemRuntimeChecks(), emitSCEVChecks(), fixupIVUsers(), fixVectorizedLoop(), getAddressAccessSCEV(), getOrCreateTripCount(), processLoopInVPlanNativePath(), llvm::LoopVectorizationCostModel::runtimeChecksRequired(), llvm::LoopVectorizationCostModel::selectInterleaveCount(), llvm::VPRecipeBuilder::tryToCreateWidenRecipe(), and widenPHIInstruction().

◆ PSI

ProfileSummaryInfo* llvm::InnerLoopVectorizer::PSI

protected

Definition at line 904 of file LoopVectorize.cpp.

Referenced by getScalarEpilogueLowering(), InnerLoopVectorizer(), and processLoopInVPlanNativePath().

◆ TLI

const TargetLibraryInfo* llvm::InnerLoopVectorizer::TLI

protected

Target Library Info.

Definition at line 811 of file LoopVectorize.cpp.

Referenced by getScalarEpilogueLowering(), llvm::LoopVectorizationCostModel::getVectorCallCost(), llvm::LoopVectorizationCostModel::getVectorIntrinsicCost(), processLoopInVPlanNativePath(), and widenCallInstruction().

◆ TripCount

Value* llvm::InnerLoopVectorizer::TripCount = nullptr

protected

Trip count of the original loop.

Definition at line 880 of file LoopVectorize.cpp.

Referenced by getOrCreateTripCount().

◆ TTI

const TargetTransformInfo* llvm::InnerLoopVectorizer::TTI

protected

Target Transform Info.

Definition at line 814 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationCostModel::calculateRegisterUsage().

◆ UF

unsigned llvm::InnerLoopVectorizer::UF

protected

The vectorization unroll factor to use.

Each scalar is vectorized to this many different vector instructions.

Definition at line 835 of file LoopVectorize.cpp.

Referenced by buildScalarSteps(), clearReductionWrapFlags(), llvm::EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(), llvm::EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(), createVectorizedLoopSkeleton(), emitMinimumIterationCountCheck(), llvm::EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(), fixFirstOrderRecurrence(), fixLCSSAPHIs(), fixReduction(), getOrCreateVectorTripCount(), llvm::LoopVectorizationPlanner::setBestPlan(), setDebugLocFromInst(), truncateToMinimalBitwidths(), vectorizeInterleaveGroup(), vectorizeMemoryInstruction(), widenCallInstruction(), widenGEP(), widenInstruction(), widenPHIInstruction(), and widenSelectInstruction().

◆ VectorLoopValueMap

VectorizerValueMap llvm::InnerLoopVectorizer::VectorLoopValueMap

protected

Maps values from the original loop to their corresponding values in the vectorized loop.

A key value can map to either vector values, scalar values or both kinds of values, depending on whether the key was vectorized and scalarized.

Definition at line 874 of file LoopVectorize.cpp.

Referenced by buildScalarSteps(), llvm::LoopVectorizationPlanner::executePlan(), fixFirstOrderRecurrence(), fixNonInductionPHIs(), fixReduction(), getOrCreateScalarValue(), getOrCreateVectorValue(), packScalarIntoVectorValue(), recordVectorLoopValueForInductionCast(), scalarizeInstruction(), setVectorValue(), truncateToMinimalBitwidths(), and widenPHIInstruction().

◆ VectorTripCount

Value* llvm::InnerLoopVectorizer::VectorTripCount = nullptr

protected

Trip count of the widened loop (TripCount - TripCount % (VF*UF))

Definition at line 883 of file LoopVectorize.cpp.

Referenced by completeLoopSkeleton(), createInductionResumeValues(), and getOrCreateVectorTripCount().

◆ VF

ElementCount llvm::InnerLoopVectorizer::VF

protected

The vectorization SIMD factor to use.

Each vector will have this many vector elements.

Definition at line 831 of file LoopVectorize.cpp.

The documentation for this class was generated from the following file:

lib/Transforms/Vectorize/LoopVectorize.cpp

Public Types

Public Member Functions

Protected Types

Protected Member Functions

Protected Attributes

Friends

Detailed Description

Member Typedef Documentation

◆ PhiVector

◆ ScalarParts

◆ VectorParts

Constructor & Destructor Documentation

◆ InnerLoopVectorizer()

◆ ~InnerLoopVectorizer()

Member Function Documentation

◆ addMetadata() [1/2]

◆ addMetadata() [2/2]

◆ addNewMetadata()

◆ areSafetyChecksAdded()

◆ buildScalarSteps()

◆ clearReductionWrapFlags()

◆ completeLoopSkeleton()

◆ createBitOrPointerCast()

◆ createInductionResumeValues()

◆ createInductionVariable()

◆ createVectorIntOrFpInductionPHI()

◆ createVectorizedLoopSkeleton()

◆ createVectorLoopSkeleton()

◆ emitMemRuntimeChecks()

◆ emitMinimumIterationCountCheck()

◆ emitSCEVChecks()

◆ emitTransformedIndex()

◆ fixCrossIterationPHIs()

◆ fixFirstOrderRecurrence()

◆ fixLCSSAPHIs()

◆ fixNonInductionPHIs()

◆ fixReduction()

◆ fixupIVUsers()

◆ fixVectorizedLoop()

◆ getBroadcastInstrs()

◆ getOrCreateScalarValue()

◆ getOrCreateTripCount()

◆ getOrCreateVectorTripCount()

◆ getOrCreateVectorValue()

◆ getStepVector()

◆ needsScalarInduction()

◆ packScalarIntoVectorValue()

◆ printDebugTracesAtEnd()

◆ printDebugTracesAtStart()

◆ recordVectorLoopValueForInductionCast()

◆ reverseVector()

◆ scalarizeInstruction()

◆ setDebugLocFromInst()

◆ setVectorValue()

◆ shouldScalarizeInstruction()

◆ sinkScalarOperands()

◆ truncateToMinimalBitwidths()

◆ vectorizeInterleaveGroup()

◆ vectorizeMemoryInstruction()

◆ widenCallInstruction()

◆ widenGEP()

◆ widenInstruction()

◆ widenIntOrFpInduction()

◆ widenPHIInstruction()

◆ widenSelectInstruction()

Friends And Related Function Documentation

◆ LoopVectorizationPlanner

Member Data Documentation

◆ AA

◆ AC

◆ AddedSafetyChecks

◆ BFI

◆ Builder

◆ Cost

◆ DT

◆ Induction

◆ IVEndValues

◆ Legal

◆ LI

◆ LoopBypassBlocks