VPlanTransforms.cpp File Reference

This file implements a set of utility VPlan to VPlan transformations. More...

#include "VPlanTransforms.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"

Go to the source code of this file.

Classes
class	SinkStoreInfo
	Helper for extra no-alias checks via known-safe recipe and SCEV. More...
struct	RemoveMask_match< Op0_t, Op1_t >

Functions
static bool	canHoistOrSinkWithNoAliasCheck (const MemoryLocation &MemLoc, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, std::optional< SinkStoreInfo > SinkInfo={})
	Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.
static Type *	getLoadStoreValueType (VPReplicateRecipe *R, bool IsLoad)
	Get the value type of the replicate load or store.
template<unsigned Opcode>
static SmallVector< SmallVector< VPReplicateRecipe *, 4 > >	collectGroupedReplicateMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L, function_ref< bool(VPReplicateRecipe *)> FilterFn)
	Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.
static bool	sinkScalarOperands (VPlan &Plan)
static VPValue *	getPredicatedMask (VPRegionBlock *R)
	If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
static VPBasicBlock *	getPredicatedThenBlock (VPRegionBlock *R)
	If R is a triangle region, return the 'then' block of the triangle.
static bool	mergeReplicateRegionsIntoSuccessors (VPlan &Plan)
static VPRegionBlock *	createReplicateRegion (VPReplicateRecipe *PredRecipe, VPRegionBlock *ParentRegion, VPlan &Plan)
static void	addReplicateRegions (VPlan &Plan)
static void	removeRedundantInductionCasts (VPlan &Plan)
	Remove redundant casts of inductions.
static VPScalarIVStepsRecipe *	createScalarIVSteps (VPlan &Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, Instruction *TruncI, VPIRValue *StartV, VPValue *Step, DebugLoc DL, VPBuilder &Builder)
static bool	isDeadRecipe (VPRecipeBase &R)
	Returns true if R is dead and can be removed.
static SmallVector< VPUser * >	collectUsersRecursively (VPValue *V)
static VPValue *	scalarizeVPWidenPointerInduction (VPWidenPointerInductionRecipe *PtrIV, VPlan &Plan, VPBuilder &Builder)
	Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)).
static void	legalizeAndOptimizeInductions (VPlan &Plan)
	Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
static VPWidenInductionRecipe *	getOptimizableIVOf (VPValue *VPV, PredicatedScalarEvolution &PSE)
	Check if VPV is an untruncated wide induction, either before or after the increment.
static VPValue *	optimizeEarlyExitInductionUser (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE)
	Attempts to optimize the induction variable exit values for users in the early exit block.
static VPValue *	tryToComputeEndValueForInduction (VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPValue *VectorTC)
	Compute the end value for WideIV, unless it is truncated.
static VPValue *	optimizeLatchExitInductionUser (VPlan &Plan, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, PredicatedScalarEvolution &PSE)
	Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
static void	removeRedundantExpandSCEVRecipes (VPlan &Plan)
	Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.
static void	recursivelyDeleteDeadRecipes (VPValue *V)
static std::optional< std::pair< bool, unsigned > >	getOpcodeOrIntrinsicID (const VPSingleDefRecipe *R)
	Get any instruction opcode or intrinsic ID data embedded in recipe R.
static VPIRValue *	tryToFoldLiveIns (VPSingleDefRecipe &R, ArrayRef< VPValue * > Operands, const DataLayout &DL, LLVMContext &Ctx)
	Try to fold R using InstSimplifyFolder.
static bool	simplifyLogicalRecipe (VPSingleDefRecipe *Def, VPBuilder &Builder, bool CanCreateNewRecipe)
	Try to simplify logical and bitwise recipes in Def.
static void	simplifyRecipe (VPSingleDefRecipe *Def)
	Try to simplify VPSingleDefRecipe Def.
static void	reassociateHeaderMask (VPlan &Plan)
	Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.
static std::optional< Instruction::BinaryOps >	getUnmaskedDivRemOpcode (Intrinsic::ID ID)
static void	narrowToSingleScalarRecipes (VPlan &Plan)
static void	removeCommonBlendMask (VPBlendRecipe *Blend)
	Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.
static void	simplifyBlends (VPlan &Plan)
	Normalize and simplify VPBlendRecipes.
static bool	optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF)
	Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
static bool	isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Return true if Cond is known to be true for given BestVF and BestUF.
static bool	tryToReplaceALMWithWideALM (VPlan &Plan, ElementCount VF, unsigned UF)
	Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.
static bool	simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Try to simplify the branch condition of Plan.
static bool	cannotHoistOrSinkRecipe (VPRecipeBase &R, VPBasicBlock *FirstBB, VPBasicBlock *LastBB)
	Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.
static void	licm (VPlan &Plan)
	Move loop-invariant recipes out of the vector loop region in Plan.
static VPActiveLaneMaskPHIRecipe *	addVPLaneMaskPhiAndUpdateExitBranch (VPlan &Plan)
template<typename Op0_t, typename Op1_t>
static RemoveMask_match< Op0_t, Op1_t >	m_RemoveMask (const Op0_t &In, Op1_t &Out)
	Match a specific mask In, or a combination of it (logical-and In, Out).
static std::optional< Intrinsic::ID >	getVPDivRemIntrinsic (Intrinsic::ID IntrID)
static VPRecipeBase *	optimizeMaskToEVL (VPValue *HeaderMask, VPRecipeBase &CurRecipe, VPValue &EVL)
	Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.
static void	fixupVFUsersForEVL (VPlan &Plan, VPValue &EVL)
	After replacing the canonical IV with a EVL-based IV, fixup recipes that use VF to use the EVL instead to avoid incorrect updates on the penultimate iteration.
static void	expandVPWidenIntOrFpInduction (VPWidenIntOrFpInductionRecipe *WidenIVR)
	Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.
static void	expandVPWidenPointerInduction (VPWidenPointerInductionRecipe *R)
	Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.
static void	expandVPDerivedIV (VPDerivedIVRecipe *R)
	Expand a VPDerivedIVRecipe into executable recipes.
static VPExpressionRecipe *	tryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static VPExpressionRecipe *	tryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
static void	tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range)
	This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
static VPIRMetadata	getCommonMetadata (ArrayRef< VPReplicateRecipe * > Recipes)
template<unsigned Opcode>
static SmallVector< SmallVector< VPReplicateRecipe *, 4 > >	collectComplementaryPredicatedMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L)
template<typename InstType>
static VPReplicateRecipe *	findRecipeWithMinAlign (ArrayRef< VPReplicateRecipe * > Group)
static bool	canSinkStoreWithNoAliasCheck (ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution &PSE, const Loop &L)
static bool	canNarrowLoad (VPSingleDefRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx, bool IsScalable)
	Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
static bool	canNarrowOps (ArrayRef< VPValue * > Ops, bool IsScalable)
static std::optional< ElementCount >	isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI)
	Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.
static bool	isAlreadyNarrow (VPValue *VPV)
	Returns true if VPValue is a narrow VPValue.
static VPValue *	narrowInterleaveGroupOp (VPValue *V, SmallPtrSetImpl< VPValue * > &NarrowedOps)
static VPWidenIntOrFpInductionRecipe *	getExpressionIV (VPValue *V)
	Check if V is a binary expression of a widened IV and a loop-invariant value.
static VPValue *	cloneBinOpForScalarIV (VPWidenRecipe *BinOp, VPValue *ScalarIV, VPWidenIntOrFpInductionRecipe *WidenIV)
	Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.
static bool	areVFParamsOk (const VFInfo &Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution &PSE, const Loop *L)
	Returns true if Info's parameter kinds are compatible with Args.
static Function *	findVectorVariant (CallInst *CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution &PSE, const Loop *L)
	Find a vector variant of CI for VF, respecting MaskRequired.
static CallWideningDecision	decideCallWidening (VPInstruction &VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext &CostCtx)
	Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.

Detailed Description

This file implements a set of utility VPlan to VPlan transformations.

Definition in file VPlanTransforms.cpp.

Function Documentation

addReplicateRegions()

void addReplicateRegions ( VPlan & Plan )

static

Definition at line 556 of file VPlanTransforms.cpp.

References llvm::VPBlockUtils::blocksOnly(), createReplicateRegion(), llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getExiting(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::RegionBase< Tr >::getParent(), llvm::VPBlockBase::getParent(), llvm::Value::hasName(), llvm::VPBlockUtils::insertOnEdge(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPRegionBlock::setExiting(), llvm::VPBasicBlock::splitAt(), llvm::SplitBlock(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

addVPLaneMaskPhiAndUpdateExitBranch()

VPActiveLaneMaskPHIRecipe * addVPLaneMaskPhiAndUpdateExitBranch ( VPlan & Plan )

static

Definition at line 2840 of file VPlanTransforms.cpp.

References llvm::VPInstruction::ActiveLaneMask, llvm::VPUser::addOperand(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::CanonicalIVIncrementForPart, llvm::VPRegionBlock::clearCanonicalIVNUW(), DL, llvm::VPRecipeBase::eraseFromParent(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPRegionBlock::getOrCreateCanonicalIVIncrement(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPlan::getVF(), and llvm::VPlan::getZero().

Referenced by llvm::VPlanTransforms::addActiveLaneMask().

areVFParamsOk()

bool areVFParamsOk ( const VFInfo & Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution & PSE, const Loop * L )

static

Returns true if Info's parameter kinds are compatible with Args.

Definition at line 6796 of file VPlanTransforms.cpp.

References llvm::all_of(), and llvm::PredicatedScalarEvolution::getSE().

Referenced by findVectorVariant().

canHoistOrSinkWithNoAliasCheck()

bool canHoistOrSinkWithNoAliasCheck ( const MemoryLocation & MemLoc, VPBasicBlock * FirstBB, VPBasicBlock * LastBB, std::optional< SinkStoreInfo > SinkInfo = {} )

static

Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.

If SinkInfo is std::nullopt, only recipes that may write to memory are checked (for load hoisting). Otherwise recipes that both read and write memory are checked, and SCEV is used to prove no-alias between the group leader and other replicate recipes (for store sinking).

Definition at line 223 of file VPlanTransforms.cpp.

Referenced by cannotHoistOrSinkRecipe(), canSinkStoreWithNoAliasCheck(), and llvm::VPlanTransforms::hoistPredicatedLoads().

canNarrowLoad()

bool canNarrowLoad ( VPSingleDefRecipe * WideMember0, unsigned OpIdx, VPValue * OpV, unsigned Idx, bool IsScalable )

static

Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.

V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group.

Definition at line 5348 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), IR, and OpIdx.

Referenced by canNarrowOps().

canNarrowOps()

bool canNarrowOps ( ArrayRef< VPValue * > Ops, bool IsScalable )

static

Definition at line 5364 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), canNarrowLoad(), canNarrowOps(), llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), getOpcodeOrIntrinsicID(), llvm::isa(), OpIdx, P, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by canNarrowOps(), and llvm::VPlanTransforms::narrowInterleaveGroups().

cannotHoistOrSinkRecipe()

bool cannotHoistOrSinkRecipe ( VPRecipeBase & R, VPBasicBlock * FirstBB, VPBasicBlock * LastBB )

static

Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.

Definition at line 2530 of file VPlanTransforms.cpp.

References canHoistOrSinkWithNoAliasCheck(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::vputils::getMemoryLocation(), and llvm::isa().

Referenced by licm().

canSinkStoreWithNoAliasCheck()

bool canSinkStoreWithNoAliasCheck ( ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution & PSE, const Loop & L )

static

Definition at line 4845 of file VPlanTransforms.cpp.

References llvm::ArrayRef< T >::back(), llvm::ArrayRef< T >::begin(), canHoistOrSinkWithNoAliasCheck(), llvm::ArrayRef< T >::end(), llvm::ArrayRef< T >::front(), and llvm::vputils::getMemoryLocation().

Referenced by llvm::VPlanTransforms::sinkPredicatedStores().

cloneBinOpForScalarIV()

VPValue * cloneBinOpForScalarIV ( VPWidenRecipe * BinOp, VPValue * ScalarIV, VPWidenIntOrFpInductionRecipe * WidenIV )

static

Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.

Definition at line 5842 of file VPlanTransforms.cpp.

References assert(), llvm::VPWidenRecipe::clone(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPWidenRecipe::getOpcode(), llvm::VPRecipeBase::insertAfter(), and llvm::Instruction::isBinaryOp().

Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().

collectComplementaryPredicatedMemOps()

template<unsigned Opcode>

SmallVector< SmallVector< VPReplicateRecipe *, 4 > > collectComplementaryPredicatedMemOps ( VPlan & Plan, PredicatedScalarEvolution & PSE, const Loop * L )

static

Definition at line 4730 of file VPlanTransforms.cpp.

References llvm::all_equal(), assert(), llvm::bind_back(), collectGroupedReplicateMemOps(), Groups, llvm::VPReplicateRecipe::isPredicated(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::map_range(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

collectGroupedReplicateMemOps()

template<unsigned Opcode>

SmallVector< SmallVector< VPReplicateRecipe *, 4 > > collectGroupedReplicateMemOps ( VPlan & Plan, PredicatedScalarEvolution & PSE, const Loop * L, function_ref< bool(VPReplicateRecipe *)> FilterFn )

static

Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.

Definition at line 264 of file VPlanTransforms.cpp.

References A(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), getLoadStoreValueType(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getVectorLoopRegion(), Groups, llvm::isa(), llvm::VPDominatorTree::properlyDominates(), llvm::stable_sort(), llvm::to_vector(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::values(), and llvm::vp_depth_first_deep().

Referenced by collectComplementaryPredicatedMemOps().

collectUsersRecursively()

SmallVector< VPUser * > collectUsersRecursively ( VPValue * V )

static

Definition at line 834 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::VPDef::definedValues(), llvm::from_range, I, and Users.

Referenced by llvm::VPlanTransforms::clearReductionWrapFlags(), legalizeAndOptimizeInductions(), and llvm::VPlanTransforms::optimizeEVLMasks().

createReplicateRegion()

VPRegionBlock * createReplicateRegion ( VPReplicateRecipe * PredRecipe, VPRegionBlock * ParentRegion, VPlan & Plan )

static

Definition at line 514 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createReplicateRegion(), llvm::VPlan::createVPBasicBlock(), llvm::drop_end(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPReplicateRecipe::getMask(), llvm::VPValue::getNumUsers(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::DebugLoc::getUnknown(), llvm::VPBlockUtils::insertTwoBlocksAfter(), llvm::VPReplicateRecipe::isSingleScalar(), llvm::VPUser::operands(), and llvm::VPValue::replaceAllUsesWith().

Referenced by addReplicateRegions().

createScalarIVSteps()

VPScalarIVStepsRecipe * createScalarIVSteps ( VPlan & Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator * FPBinOp, Instruction * TruncI, VPIRValue * StartV, VPValue * Step, DebugLoc DL, VPBuilder & Builder )

static

Definition at line 671 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), DL, llvm::VPRegionBlock::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getScalarSizeInBits(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPBlockBase::getSingleHierarchicalPredecessor(), llvm::Value::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), and llvm::Type::isIntegerTy().

Referenced by legalizeAndOptimizeInductions(), llvm::VPlanTransforms::replaceWideCanonicalIVWithWideIV(), and scalarizeVPWidenPointerInduction().

decideCallWidening()

CallWideningDecision decideCallWidening ( VPInstruction & VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext & CostCtx )

static

Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.

Definition at line 6858 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::VPReplicateRecipe::computeCallCost(), llvm::VPWidenCallRecipe::computeCallCost(), llvm::VPWidenIntrinsicRecipe::computeCallCost(), findVectorVariant(), llvm::InstructionCost::getInvalid(), llvm::VPValue::getLiveInIRValue(), llvm::VPInstruction::getNumOperandsWithoutMask(), llvm::VPUser::getOperand(), llvm::VPRecipeValue::getScalarType(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::getVectorIntrinsicIDForCall(), IntrinsicCost, llvm::VPCostContext::isFreeScalarIntrinsic(), llvm::VPCostContext::isMaskRequired(), llvm::ElementCount::isScalar(), llvm::VPCostContext::L, llvm::VPCostContext::PSE, llvm::VPCostContext::TLI, and llvm::VPCostContext::willBeScalarized().

Referenced by llvm::VPlanTransforms::makeCallWideningDecisions().

expandVPDerivedIV()

void expandVPDerivedIV ( VPDerivedIVRecipe * R )

static

Expand a VPDerivedIVRecipe into executable recipes.

Definition at line 3904 of file VPlanTransforms.cpp.

References assert(), llvm::FMul, llvm::DebugLoc::getCompilerGenerated(), llvm::FPMathOperator::getFastMathFlags(), llvm::Operator::getOpcode(), llvm::Type::getScalarType(), llvm::VPValue::getScalarType(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), and llvm_unreachable.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

expandVPWidenIntOrFpInduction()

void expandVPWidenIntOrFpInduction ( VPWidenIntOrFpInductionRecipe * WidenIVR )

static

Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge value.

In the following example:

vector.ph: Successor(s): vector loop

<x1> vector loop: { vector.body: WIDEN-INDUCTION i = phi start, step, vf ... EMIT branch-on-count ... No successors }

WIDEN-INDUCTION will get expanded to:

vector.ph: ... vp<induction.start> = ... vp<induction.increment> = ...

Successor(s): vector loop

<x1> vector loop: { vector.body: ir<i> = WIDEN-PHI vp<induction.start>, vp<vec.ind.next> ... vp<vec.ind.next> = add ir<i>, vp<induction.increment> EMIT branch-on-count ... No successors }

Definition at line 3754 of file VPlanTransforms.cpp.

References assert(), llvm::VPInstruction::Broadcast, llvm::VPBuilder::createWidenPhi(), DL, llvm::IntegerType::get(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPWidenIntOrFpInductionRecipe::getLastUnrolledPartOperand(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getPlan(), llvm::Type::getScalarSizeInBits(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenIntOrFpInductionRecipe::getSplatVFValue(), llvm::VPWidenIntOrFpInductionRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPBasicBlock::getTerminator(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::InductionDescriptor::IK_IntInduction, llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Next, llvm::VPValue::replaceAllUsesWith(), and llvm::VPInstruction::StepVector.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

expandVPWidenPointerInduction()

void expandVPWidenPointerInduction ( VPWidenPointerInductionRecipe * R )

static

Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value, phi and backedge value.

In the following example:

<x1> vector loop: { vector.body: EMIT ir<ptr.iv> = WIDEN-POINTER-INDUCTION start, step, vf ... EMIT branch-on-count ... }

WIDEN-POINTER-INDUCTION will get expanded to:

<x1> vector loop: { vector.body: EMIT-SCALAR pointer.phi = phi start, ptr.ind EMIT mul = mul stepvector, step EMIT vector.gep = wide-ptradd pointer.phi, mul ... EMIT ptr.ind = ptradd pointer.phi, vf EMIT branch-on-count ... }

Definition at line 3863 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), DL, llvm::VPBlockBase::getExitingBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPValue::getScalarType(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::InductionDescriptor::IK_PtrInduction, llvm::Offset, and llvm::VPInstruction::StepVector.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes().

findRecipeWithMinAlign()

template<typename InstType>

VPReplicateRecipe * findRecipeWithMinAlign ( ArrayRef< VPReplicateRecipe * > Group )

static

Definition at line 4788 of file VPlanTransforms.cpp.

References A(), B(), llvm::cast(), and llvm::min_element().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

findVectorVariant()

Function * findVectorVariant ( CallInst * CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution & PSE, const Loop * L )

static

Find a vector variant of CI for VF, respecting MaskRequired.

Returns the variant function, or nullptr. Masked variants are assumed to take the mask as a trailing parameter.

Definition at line 6823 of file VPlanTransforms.cpp.

References areVFParamsOk(), llvm::find_if(), llvm::Module::getFunction(), llvm::VFDatabase::getMappings(), llvm::Instruction::getModule(), and llvm::CallBase::isNoBuiltin().

Referenced by decideCallWidening().

fixupVFUsersForEVL()

void fixupVFUsersForEVL ( VPlan & Plan, VPValue & EVL )

static

After replacing the canonical IV with a EVL-based IV, fixup recipes that use VF to use the EVL instead to avoid incorrect updates on the penultimate iteration.

Definition at line 3150 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), llvm::VPBuilder::createScalarZExtOrTrunc(), llvm::vputils::findHeaderMask(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPlan::getContext(), llvm::VPValue::getDefiningRecipe(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getInt32Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPlan::getMiddleBlock(), llvm::VPlan::getOrAddLiveIn(), llvm::VPRecipeBase::getParent(), llvm::VPRegionBlock::getPreheaderVPBB(), llvm::VPValue::getScalarType(), llvm::ConstantInt::getSigned(), llvm::VPBuilder::getToInsertAfter(), llvm::VPlan::getTrue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_ULT, llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::IsaPred, llvm::PatternMatch::m_c_Add(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::PatternMatchHelpers::m_Isa(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::VPValue::replaceAllUsesWith(), llvm::VPValue::replaceUsesWithIf(), llvm::VPInstruction::StepVector, llvm::VPValue::users(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::addExplicitVectorLength().

getCommonMetadata()

VPIRMetadata getCommonMetadata ( ArrayRef< VPReplicateRecipe * > Recipes )

static

Definition at line 4721 of file VPlanTransforms.cpp.

References llvm::drop_begin(), llvm::ArrayRef< T >::front(), and llvm::VPIRMetadata::intersect().

Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().

getExpressionIV()

VPWidenIntOrFpInductionRecipe * getExpressionIV ( VPValue * V )

static

Check if V is a binary expression of a widened IV and a loop-invariant value.

Returns the widened IV if found, nullptr otherwise.

Definition at line 5823 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::VPValue::isDefinedOutsideLoopRegions(), llvm::Instruction::isIntDivRem(), and std::swap().

Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().

getLoadStoreValueType()

Type * getLoadStoreValueType ( VPReplicateRecipe * R, bool IsLoad )

static

Get the value type of the replicate load or store.

IsLoad indicates whether it is a load.

Definition at line 255 of file VPlanTransforms.cpp.

References llvm::Type::getScalarType().

Referenced by collectGroupedReplicateMemOps().

getOpcodeOrIntrinsicID()

std::optional< std::pair< bool, unsigned > > getOpcodeOrIntrinsicID ( const VPSingleDefRecipe * R )

static

Get any instruction opcode or intrinsic ID data embedded in recipe R.

Returns an optional pair, where the first element indicates whether it is an intrinsic ID.

Definition at line 1242 of file VPlanTransforms.cpp.

References llvm::Default, I, and llvm::VPInstruction::OpsEnd.

Referenced by canNarrowOps(), and tryToFoldLiveIns().

getOptimizableIVOf()

VPWidenInductionRecipe * getOptimizableIVOf ( VPValue * VPV, PredicatedScalarEvolution & PSE )

static

Check if VPV is an untruncated wide induction, either before or after the increment.

If so return the header IV (before the increment), otherwise return null.

Definition at line 960 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm_unreachable, llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_FAdd(), llvm::VPlanPatternMatch::m_GetElementPtr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().

getPredicatedMask()

VPValue * getPredicatedMask ( VPRegionBlock * R )

static

If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.

Definition at line 393 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::dyn_cast(), and llvm::isa().

Referenced by mergeReplicateRegionsIntoSuccessors().

getPredicatedThenBlock()

VPBasicBlock * getPredicatedThenBlock ( VPRegionBlock * R )

static

If R is a triangle region, return the 'then' block of the triangle.

Definition at line 403 of file VPlanTransforms.cpp.

References llvm::cast(), and llvm::dyn_cast().

Referenced by mergeReplicateRegionsIntoSuccessors().

getUnmaskedDivRemOpcode()

std::optional< Instruction::BinaryOps > getUnmaskedDivRemOpcode ( Intrinsic::ID ID )

static

Definition at line 1866 of file VPlanTransforms.cpp.

Referenced by narrowToSingleScalarRecipes().

getVPDivRemIntrinsic()

std::optional< Intrinsic::ID > getVPDivRemIntrinsic ( Intrinsic::ID IntrID )

static

Definition at line 2947 of file VPlanTransforms.cpp.

Referenced by optimizeMaskToEVL().

isAlreadyNarrow()

bool isAlreadyNarrow ( VPValue * VPV )

static

Returns true if VPValue is a narrow VPValue.

Definition at line 5443 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), and llvm::isa().

Referenced by narrowInterleaveGroupOp(), and llvm::VPlanTransforms::narrowInterleaveGroups().

isConditionTrueViaVFAndUF()

bool isConditionTrueViaVFAndUF ( VPValue * Cond, VPlan & Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution & PSE )

static

Return true if Cond is known to be true for given BestVF and BestUF.

Definition at line 2152 of file VPlanTransforms.cpp.

References llvm::any_of(), assert(), llvm::CallingConv::C, Cond, llvm::VPRegionBlock::getCanonicalIV(), llvm::ScalarEvolution::getElementCount(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::isa(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by simplifyBranchConditionForVFAndUF().

isConsecutiveInterleaveGroup()

std::optional< ElementCount > isConsecutiveInterleaveGroup ( VPInterleaveRecipe * InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo & TTI )

static

Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.

The interleave group must also access the full vector width.

Definition at line 5399 of file VPlanTransforms.cpp.

References llvm::all_of(), assert(), llvm::VPDef::definedValues(), llvm::VPInterleaveBase::getInterleaveGroup(), llvm::VPInterleaveBase::getMask(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getScalarType(), llvm::VPInterleaveBase::getStoredValues(), llvm::VPDef::getVPValue(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::TargetTransformInfo::RGK_ScalableVector, and Size.

Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().

isDeadRecipe()

bool isDeadRecipe ( VPRecipeBase & R )

static

Returns true if R is dead and can be removed.

Definition at line 787 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::dyn_cast(), llvm::PatternMatch::m_Intrinsic(), and llvm::PatternMatch::match().

Referenced by narrowToSingleScalarRecipes(), recursivelyDeleteDeadRecipes(), and llvm::VPlanTransforms::removeDeadRecipes().

legalizeAndOptimizeInductions()

void legalizeAndOptimizeInductions ( VPlan & Plan )

static

Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.

If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.

Definition at line 873 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), collectUsersRecursively(), createScalarIVSteps(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::vputils::isSingleScalar(), llvm::VPlanPatternMatch::m_Binary(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::reverse(), scalarizeVPWidenPointerInduction(), and Users.

Referenced by llvm::VPlanTransforms::optimize().

licm()

void licm ( VPlan & Plan )

static

Move loop-invariant recipes out of the vector loop region in Plan.

Definition at line 2541 of file VPlanTransforms.cpp.

References llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), cannotHoistOrSinkRecipe(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), llvm::dyn_cast(), llvm::VPBasicBlock::end(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::make_early_inc_range(), llvm::VPDominatorTree::properlyDominates(), llvm::reverse(), and llvm::vp_depth_first_shallow().

m_RemoveMask()

template<typename Op0_t, typename Op1_t>

RemoveMask_match< Op0_t, Op1_t > m_RemoveMask ( const Op0_t & In, Op1_t & Out )

inlinestatic

Match a specific mask In, or a combination of it (logical-and In, Out).

Returns the remaining part Out if so, or nullptr otherwise.

Definition at line 2942 of file VPlanTransforms.cpp.

Referenced by optimizeMaskToEVL().

mergeReplicateRegionsIntoSuccessors()

bool mergeReplicateRegionsIntoSuccessors ( VPlan & Plan )

static

Definition at line 425 of file VPlanTransforms.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallPtrSetImplBase::empty(), llvm::VPlan::getEntry(), llvm::VPBasicBlock::getFirstNonPhi(), getPredicatedMask(), getPredicatedThenBlock(), llvm::VPBlockBase::getSingleSuccessor(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

narrowInterleaveGroupOp()

VPValue * narrowInterleaveGroupOp ( VPValue * V, SmallPtrSetImpl< VPValue * > & NarrowedOps )

static

Definition at line 5453 of file VPlanTransforms.cpp.

References assert(), llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::dyn_cast(), E(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), isAlreadyNarrow(), N, and narrowInterleaveGroupOp().

Referenced by narrowInterleaveGroupOp().

narrowToSingleScalarRecipes()

void narrowToSingleScalarRecipes ( VPlan & Plan )

static

Definition at line 1881 of file VPlanTransforms.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastLane, llvm::VPInstruction::ExtractLastPart, llvm::VPInstruction::ExtractPenultimateElement, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPlan::getEntry(), getUnmaskedDivRemOpcode(), llvm::VPlan::hasScalarVFOnly(), llvm::is_contained(), llvm::isa(), isDeadRecipe(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::make_early_inc_range(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), Opc, llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::optimize().

optimizeEarlyExitInductionUser()

VPValue * optimizeEarlyExitInductionUser ( VPlan & Plan, VPValue * Op, PredicatedScalarEvolution & PSE )

static

Attempts to optimize the induction variable exit values for users in the early exit block.

Definition at line 1017 of file VPlanTransforms.cpp.

References B(), llvm::cast(), DL, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), getOptimizableIVOf(), llvm::VPValue::getScalarType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_FirstActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

optimizeLatchExitInductionUser()

VPValue * optimizeLatchExitInductionUser ( VPlan & Plan, VPValue * Op, DenseMap< VPValue *, VPValue * > & EndValues, PredicatedScalarEvolution & PSE )

static

Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.

Definition at line 1101 of file VPlanTransforms.cpp.

References assert(), B(), llvm::cast(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), getOptimizableIVOf(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getZero(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm_unreachable, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

optimizeMaskToEVL()

VPRecipeBase * optimizeMaskToEVL ( VPValue * HeaderMask, VPRecipeBase & CurRecipe, VPValue & EVL )

static

Try to optimize a CurRecipe masked by HeaderMask to a corresponding EVL-based recipe without the header mask.

Returns nullptr if no EVL-based recipe could be created. HeaderMask Header Mask. CurRecipe Recipe to be transform. EVL The explicit vector length parameter of vector-predication intrinsics.

Adjust any end pointers so that they point to the end of EVL lanes not VF.

Definition at line 2969 of file VPlanTransforms.cpp.

References llvm::cast(), llvm::VPBuilder::createScalarZExtOrTrunc(), DL, llvm::dyn_cast(), llvm::VPlan::getConstantInt(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getPlan(), llvm::VPValue::getScalarType(), llvm::VPlan::getTrue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVF(), getVPDivRemIntrinsic(), llvm::VPDef::getVPSingleValue(), LHS, llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_Intrinsic(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_MaskedLoad(), llvm::PatternMatch::m_MaskedStore(), m_RemoveMask(), llvm::VPlanPatternMatch::m_Reverse(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_TruncOrSelf(), llvm::VPlanPatternMatch::m_VecEndPtr(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::Reverse, and RHS.

Referenced by llvm::VPlanTransforms::optimizeEVLMasks().

optimizeVectorInductionWidthForTCAndVFUF()

bool optimizeVectorInductionWidthForTCAndVFUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF )

static

Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.

Definition at line 2078 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::cast(), llvm::IntegerType::get(), llvm::APInt::getActiveBits(), llvm::VPlan::getBackedgeTakenCount(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getSingleUser(), llvm::VPlan::getTripCount(), llvm::VPWidenIntOrFpInductionRecipe::getTruncInst(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::VPlan::getZero(), llvm::VPValue::hasMoreThanOneUniqueUser(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::PowerOf2Ceil(), llvm::APIntOps::RoundingUDiv(), and llvm::APInt::UP.

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

reassociateHeaderMask()

void reassociateHeaderMask ( VPlan & Plan )

static

Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.

in optimizeEVLMasks.

Definition at line 1841 of file VPlanTransforms.cpp.

References llvm::append_range(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::vputils::findHeaderMask(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::VPValue::users(), users, X, and Y.

Referenced by llvm::VPlanTransforms::optimize().

recursivelyDeleteDeadRecipes()

void recursivelyDeleteDeadRecipes ( VPValue * V )

static

Definition at line 1219 of file VPlanTransforms.cpp.

References llvm::append_range(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPValue::getDefiningRecipe(), llvm::SmallPtrSetImpl< PtrType >::insert(), isDeadRecipe(), llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::VPlanTransforms::optimizeEVLMasks(), and simplifyBlends().

removeCommonBlendMask()

void removeCommonBlendMask ( VPBlendRecipe * Blend )

static

Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.

Definition at line 1980 of file VPlanTransforms.cpp.

References llvm::VPValue::getDefiningRecipe(), llvm::VPBlendRecipe::getMask(), llvm::VPBlendRecipe::getNumIncomingValues(), llvm::VPUser::getOperand(), I, llvm::VPBlendRecipe::isNormalized(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPBlendRecipe::setMask().

Referenced by simplifyBlends().

removeRedundantExpandSCEVRecipes()

void removeRedundantExpandSCEVRecipes ( VPlan & Plan )

static

Remove redundant ExpandSCEVRecipes in Plan's entry block by replacing them with already existing recipes expanding the same SCEV expression.

Definition at line 1198 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getTripCount(), llvm::make_early_inc_range(), llvm::VPlan::resetTripCount(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().

Referenced by llvm::VPlanTransforms::optimize().

removeRedundantInductionCasts()

void removeRedundantInductionCasts ( VPlan & Plan )

static

Remove redundant casts of inductions.

Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.

Definition at line 635 of file VPlanTransforms.cpp.

References llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().

Referenced by llvm::VPlanTransforms::optimize().

scalarizeVPWidenPointerInduction()

VPValue * scalarizeVPWidenPointerInduction ( VPWidenPointerInductionRecipe * PtrIV, VPlan & Plan, VPBuilder & Builder )

static

Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)).

This is used when the recipe only generates scalar values.

Definition at line 848 of file VPlanTransforms.cpp.

References createScalarIVSteps(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPUser::getOperand(), llvm::VPWidenInductionRecipe::getStartValue(), llvm::VPlan::getZero(), and llvm::InductionDescriptor::IK_IntInduction.

Referenced by llvm::VPlanTransforms::convertToConcreteRecipes(), and legalizeAndOptimizeInductions().

simplifyBlends()

void simplifyBlends ( VPlan & Plan )

static

Normalize and simplify VPBlendRecipes.

Should be run after simplifyRecipes to make sure the masks are simplified.

Simplify BLEND a, b, Not(mask) -> BLEND b, a, mask.

Definition at line 1997 of file VPlanTransforms.cpp.

References llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::cast_or_null(), llvm::dyn_cast(), llvm::VPRegionBlock::getEntry(), llvm::VPValue::getNumUsers(), llvm::VPlan::getVectorLoopRegion(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPlanPatternMatch::m_False(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), recursivelyDeleteDeadRecipes(), removeCommonBlendMask(), llvm::VPValue::replaceAllUsesWith(), llvm::SmallPtrSetImplBase::size(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::optimize().

simplifyBranchConditionForVFAndUF()

bool simplifyBranchConditionForVFAndUF ( VPlan & Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution & PSE )

static

Try to simplify the branch condition of Plan.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 2285 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, llvm::CallingConv::C, Cond, llvm::ScalarEvolution::getElementCount(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_ULE, llvm::isa(), isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::PatternMatch::m_Add(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().

simplifyLogicalRecipe()

bool simplifyLogicalRecipe ( VPSingleDefRecipe * Def, VPBuilder & Builder, bool CanCreateNewRecipe )

static

Try to simplify logical and bitwise recipes in Def.

Definition at line 1335 of file VPlanTransforms.cpp.

References llvm::CallingConv::C, llvm::VPlan::getAllOnesValue(), llvm::VPlan::getFalse(), llvm::VPlan::getZero(), llvm::PatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_c_BinaryAnd(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_LogicalAnd(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_False(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_Not(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Select(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), X, and Y.

Referenced by simplifyRecipe().

simplifyRecipe()

void simplifyRecipe ( VPSingleDefRecipe * Def )

static

Try to simplify VPSingleDefRecipe Def.

Definition at line 1463 of file VPlanTransforms.cpp.

References A(), llvm::all_equal(), llvm::all_of(), llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::Broadcast, llvm::CallingConv::C, llvm::cast(), DL, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::CmpInst::FCMP_UNO, llvm::VPlanPatternMatch::findUserOf(), llvm::VPlan::getConcreteUF(), llvm::VPlan::getContext(), llvm::VPlan::getDataLayout(), llvm::CmpInst::getInversePredicate(), llvm::VPUser::getOperand(), llvm::Type::getScalarSizeInBits(), llvm::VPDef::getVPSingleValue(), llvm::VPlan::getZero(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::APInt::isPowerOf2(), llvm::vputils::isSingleScalar(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_Mul(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ComputeReductionResult(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_DerivedIV(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLane(), llvm::VPlanPatternMatch::m_ExtractLastPart(), llvm::VPlanPatternMatch::m_ExtractPenultimateElement(), llvm::PatternMatch::m_Intrinsic(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificCmp(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_True(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_UDiv(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPIRValue(), llvm::VPlanPatternMatch::m_VPPhi(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::vputils::onlyFirstLaneUsed(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceAllUsesWith(), simplifyLogicalRecipe(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::to_vector(), tryToFoldLiveIns(), X, and Y.

Referenced by llvm::VPlanTransforms::simplifyRecipes().

sinkScalarOperands()

bool sinkScalarOperands ( VPlan & Plan )

static

Definition at line 299 of file VPlanTransforms.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), Changed, llvm::VPSingleDefRecipe::clone(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::VPlan::hasScalarVFOnly(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::make_filter_range(), llvm::VPRecipeBase::moveBefore(), llvm::VPUser::operands(), llvm::VPValue::replaceUsesWithIf(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::VPValue::users(), and llvm::vp_depth_first_deep().

Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().

tryToComputeEndValueForInduction()

VPValue * tryToComputeEndValueForInduction ( VPWidenInductionRecipe * WideIV, VPBuilder & VectorPHBuilder, VPValue * VectorTC )

static

Compute the end value for WideIV, unless it is truncated.

Creates a VPDerivedIVRecipe for non-canonical inductions.

Definition at line 1067 of file VPlanTransforms.cpp.

References llvm::VPBuilder::createDerivedIV(), llvm::VPBuilder::createScalarCast(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), and llvm::PatternMatch::match().

Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().

tryToCreateAbstractReductionRecipe()

void tryToCreateAbstractReductionRecipe ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.

Definition at line 4641 of file VPlanTransforms.cpp.

References assert(), llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().

Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().

tryToFoldLiveIns()

VPIRValue * tryToFoldLiveIns ( VPSingleDefRecipe & R, ArrayRef< VPValue * > Operands, const DataLayout & DL, LLVMContext & Ctx )

static

Try to fold R using InstSimplifyFolder.

Will succeed and return a non-nullptr VPValue for a handled opcode or intrinsic ID if corresponding Operands are foldable live-ins.

Definition at line 1265 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::cast(), DL, llvm::drop_begin(), GEP, llvm::Constant::getAllOnesValue(), llvm::Type::getInt8Ty(), llvm::Constant::getNullValue(), getOpcodeOrIntrinsicID(), getType(), llvm::Instruction::isBinaryOp(), llvm::Instruction::isCast(), llvm::VPInstruction::LogicalAnd, llvm::VPlanPatternMatch::m_LiveIn(), llvm::PatternMatch::match(), llvm::VPInstruction::Not, llvm::VPInstruction::PtrAdd, and llvm::VPInstruction::WidePtrAdd.

Referenced by llvm::VPlanTransforms::simplifyKnownEVL(), and simplifyRecipe().

tryToMatchAndCreateExtendedReduction()

VPExpressionRecipe * tryToMatchAndCreateExtendedReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created recipe must be decomposed to its constituent recipes before execution.

Definition at line 4422 of file VPlanTransforms.cpp.

References A(), assert(), llvm::cast(), CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::InstructionCost::getInvalid(), llvm::RecurrenceDescriptor::getOpcode(), llvm::Type::getScalarType(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatchHelpers::m_Isa(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Range, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToMatchAndCreateMulAccumulateReduction()

VPExpressionRecipe * tryToMatchAndCreateMulAccumulateReduction ( VPReductionRecipe * Red, VPCostContext & Ctx, VFRange & Range )

static

This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.

The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))). reduce.fadd(fmul(ext(A), ext(B)))

Definition at line 4474 of file VPlanTransforms.cpp.

References A(), assert(), B(), llvm::canConstantBeExtended(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::RecurrenceDescriptor::getOpcode(), llvm::VPWidenCastRecipe::getOpcode(), llvm::VPUser::getOperand(), llvm::TargetTransformInfo::getPartialReductionExtendKind(), llvm::Type::getScalarType(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Mul, Range, llvm::Sub, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().

Referenced by tryToCreateAbstractReductionRecipe().

tryToReplaceALMWithWideALM()

bool tryToReplaceALMWithWideALM ( VPlan & Plan, ElementCount VF, unsigned UF )

static

Try to replace multiple active lane masks used for control flow with a single, wide active lane mask instruction followed by multiple extract subvector intrinsics.

This applies to the active lane mask instructions both in the loop and in the preheader. Incoming values of all ActiveLaneMaskPHIs are updated to use the new extracts from the first active lane mask, which has it's last operand (multiplier) set to UF.

Definition at line 2191 of file VPlanTransforms.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::VPInstruction::ActiveLaneMask, llvm::all_of(), assert(), llvm::VPBasicBlock::back(), llvm::cast(), DL, llvm::dyn_cast(), llvm::EnableWideActiveLaneMask, llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::Type::getInt1Ty(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPRecipeBase::insertAfter(), llvm::ElementCount::isVector(), llvm::VPlanPatternMatch::m_ActiveLaneMask(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::not_equal_to().

Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().