This file implements a set of utility VPlan to VPlan transformations. More...
#include "VPlanTransforms.h"#include "VPRecipeBuilder.h"#include "VPlan.h"#include "VPlanAnalysis.h"#include "VPlanCFG.h"#include "VPlanDominatorTree.h"#include "VPlanHelpers.h"#include "VPlanPatternMatch.h"#include "VPlanUtils.h"#include "VectorizeOptions.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/PostOrderIterator.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/SmallPtrSet.h"#include "llvm/ADT/TypeSwitch.h"#include "llvm/Analysis/IVDescriptors.h"#include "llvm/Analysis/Loads.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"#include "llvm/Analysis/ScopedNoAliasAA.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/MDBuilder.h"#include "llvm/IR/Metadata.h"#include "llvm/IR/ProfDataUtils.h"#include "llvm/Support/Casting.h"#include "llvm/Support/Regex.h"#include "llvm/Support/TypeSize.h"#include "llvm/Transforms/Utils/LoopUtils.h"Go to the source code of this file.
Classes | |
| class | SinkStoreInfo |
| Helper for extra no-alias checks via known-safe recipe and SCEV. More... | |
| struct | EarlyExitInfo |
Typedefs | |
| using | ExtendKind = TTI::PartialReductionExtendKind |
Functions | |
| static VPIRMetadata | getMetadataOf (VPRecipeBase *R) |
Returns the metadata attached to R, or an empty set for a recipe that does not carry any. | |
| static bool | canHoistOrSinkWithNoAliasCheck (const MemoryLocation &MemLoc, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, std::optional< SinkStoreInfo > SinkInfo={}) |
Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB. | |
| static Type * | getLoadStoreValueType (VPSingleDefRecipe *R) |
| Get the value type of a VPInstruction or VPReplicateRecipe load or store. | |
| template<unsigned Opcode> | |
| static SmallVector< SmallVector< VPReplicateRecipe *, 4 > > | collectGroupedReplicateMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L, function_ref< bool(VPReplicateRecipe *)> FilterFn) |
Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan. | |
| static bool | sinkScalarOperands (VPlan &Plan) |
| static VPBasicBlock * | getPredicatedThenBlock (VPRegionBlock *R) |
If R is a triangle region, return the 'then' block of the triangle. | |
| static bool | mergeReplicateRegionsIntoSuccessors (VPlan &Plan) |
| static VPRegionBlock * | createReplicateRegion (VPReplicateRecipe *PredRecipe, VPRegionBlock *ParentRegion, VPlan &Plan) |
| static void | addReplicateRegions (VPlan &Plan) |
| static void | removeRedundantInductionCasts (VPlan &Plan) |
| Remove redundant casts of inductions. | |
| static bool | tryToRemoveDeadCycle (VPRecipeBase *R) |
| If R is a phi-like recipe starting a dead cycle of recipes, erase all reachable recipes of the dead cycle and return true. | |
| static void | legalizeAndOptimizeInductions (VPlan &Plan) |
| Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only. | |
| static VPWidenInductionRecipe * | getOptimizableIVOf (VPValue *VPV, PredicatedScalarEvolution &PSE) |
Check if VPV is an untruncated wide induction, either before or after the increment. | |
| static VPValue * | optimizeEarlyExitInductionUser (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE) |
| Attempts to optimize the induction variable exit values for users in the early exit block. | |
| static VPValue * | tryToComputeEndValueForInduction (VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPValue *VectorTC) |
Compute the end value for WideIV, unless it is truncated. | |
| static VPValue * | optimizeLatchExitInductionUser (VPlan &Plan, VPValue *Op, DenseMap< VPValue *, VPValue * > &EndValues, PredicatedScalarEvolution &PSE) |
| Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop. | |
| static VPValue * | optimizeLatchExitIVUserViaSCEV (VPlan &Plan, VPValue *Op, PredicatedScalarEvolution &PSE, VPValue *ResumeTC, const Loop *L) |
| static VPValue * | simplifyLogicalRecipe (VPlan &Plan, VPSingleDefRecipe *Def) |
Try to simplify logical and bitwise recipes in Def. | |
| static void | swapSelectBranchWeights (VPRecipeBase &R, VPlan &Plan) |
Swap the branch weights recorded for R, a select whose two selected operands are being swapped,. | |
| static VPValue * | simplifyRecipe (VPlan &Plan, VPSingleDefRecipe *Def) |
Return an existing value or a live in for VPSingleDefRecipe Def if possible. | |
| static bool | isAvailableAtEndOf (VPValue *V, const VPBasicBlock *VPBB) |
Returns true if V is available at the end of VPBB, i.e. | |
| static VPSingleDefRecipe * | combineRecipe (VPlan &Plan, VPSingleDefRecipe *Def, VPCombineBuilder &Builder) |
Combine Def into a simpler recipe. | |
| static void | reassociateHeaderMask (VPlan &Plan) |
| Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g. | |
| static std::optional< Instruction::BinaryOps > | getUnmaskedDivRemOpcode (Intrinsic::ID ID) |
| static void | narrowToSingleScalarRecipes (VPlan &Plan) |
| static void | removeCommonBlendMask (VPBlendRecipe *Blend) |
Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks. | |
| static void | simplifyBlends (VPlan &Plan) |
| Normalize and simplify VPBlendRecipes. | |
| static bool | optimizeVectorInductionWidthForTCAndVFUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF) |
Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF. | |
| static bool | isConditionTrueViaVFAndUF (VPValue *Cond, VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE) |
Return true if Cond is known to be true for given BestVF and BestUF. | |
| static bool | replaceMaskWithCompareForScalarPlan (VPlan &Plan, ElementCount BestVF) |
| static bool | simplifyBranchConditionForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE) |
Try to simplify the branch condition of Plan. | |
| static bool | cannotHoistOrSinkRecipe (VPRecipeBase &R, VPBasicBlock *FirstBB, VPBasicBlock *LastBB, bool Sinking=false) |
Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region. | |
| static void | licm (VPlan &Plan, const VPDominatorTree &VPDT) |
Move loop-invariant recipes out of the vector loop region in Plan. | |
| static auto | m_Uncountable (VPValue *&Cond) |
| Matches an exit condition formed by comparing a value loaded from memory with a loop-invariant term. | |
| static auto | m_Countable (VPValue *&Cmp, PredicatedScalarEvolution &PSE, Loop *L) |
| Matches an exit condition formed by comparing the current value of a affine add recurrence in the given loop with a stride of 1 against a loop-invariant term. | |
| static VPValue * | getRecipesForUncountableExit (SmallVectorImpl< VPInstruction * > &Recipes, VPBasicBlock *LatchVPBB) |
| Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison. | |
| static bool | handleUncountableExitsWithSideEffects (VPlan &Plan, SmallVectorImpl< EarlyExitInfo > &Exits, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) |
Update Plan to mask memory operations in the loop based on whether the early exit is taken or not. | |
| static bool | loopMayHaveSideEffects (VPBasicBlock *HeaderVPBB) |
| static VPExpressionRecipe * | tryToMatchAndCreateExtendedReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid. | |
| static ExtendKind | getPartialReductionExtendKind (VPWidenCastRecipe *Cast) |
| static bool | canFoldOuterExtendIntoMul (VPWidenRecipe *Mul) |
Returns true if the narrow Mul in ext(mul(ext(A), ext(B))) or ext(mul(ext(A), C)) cannot wrap. | |
| static VPExpressionRecipe * | tryToMatchAndCreateMulAccumulateReduction (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid. | |
| static void | tryToCreateAbstractReductionRecipe (VPReductionRecipe *Red, VPCostContext &Ctx, VFRange &Range) |
| This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation. | |
| static VPIRMetadata | getCommonMetadata (ArrayRef< VPReplicateRecipe * > Recipes) |
| template<unsigned Opcode> | |
| static SmallVector< SmallVector< VPReplicateRecipe *, 4 > > | collectComplementaryPredicatedMemOps (VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) |
| template<typename InstType> | |
| static VPReplicateRecipe * | findRecipeWithMinAlign (ArrayRef< VPReplicateRecipe * > Group) |
| static bool | canSinkStoreWithNoAliasCheck (ArrayRef< VPReplicateRecipe * > StoresToSink, PredicatedScalarEvolution &PSE, const Loop &L) |
| static bool | canNarrowLoad (VPSingleDefRecipe *WideMember0, unsigned OpIdx, VPValue *OpV, unsigned Idx, bool IsScalable) |
Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe. | |
| static bool | canNarrowOps (ArrayRef< VPValue * > Ops, bool IsScalable, DenseMap< VPValue *, SmallVector< VPValue * > > &FirstMembersOf) |
Returns true if the wide recipes in Ops and their operand trees can be narrowed. | |
| static std::optional< ElementCount > | isConsecutiveInterleaveGroup (VPInterleaveRecipe *InterleaveR, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI) |
Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF. | |
| static bool | isAlreadyNarrow (VPValue *VPV) |
Returns true if VPValue is a narrow VPValue. | |
| static VPValue * | narrowInterleaveGroupOp (ArrayRef< VPValue * > Members, SmallPtrSetImpl< VPValue * > &NarrowedOps, VPBasicBlock *Preheader) |
| static VPWidenIntOrFpInductionRecipe * | getExpressionIV (VPValue *V) |
Check if V is a binary expression of a widened IV and a loop-invariant value. | |
| static VPValue * | cloneBinOpForScalarIV (VPWidenRecipe *BinOp, VPValue *ScalarIV, VPWidenIntOrFpInductionRecipe *WidenIV) |
Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe. | |
| static std::optional< bool > | getStepDirection (const SCEV *S, ScalarEvolution &SE) |
If S is an affine AddRec, returns true if its step is known to be positive and false if it is known to be negative. | |
| static void | updatePartialReductionPhiAndResult (VPlan &Plan, VPReductionPHIRecipe *Phi, unsigned Factor, RecurKind RK) |
| static bool | areVFParamsOk (const VFInfo &Info, ArrayRef< VPValue * > Args, PredicatedScalarEvolution &PSE, const Loop *L) |
Returns true if Info's parameter kinds are compatible with Args. | |
| static Function * | findVectorVariant (CallInst *CI, ArrayRef< VPValue * > Args, ElementCount VF, bool MaskRequired, PredicatedScalarEvolution &PSE, const Loop *L) |
Find a vector variant of CI for VF, respecting MaskRequired. | |
| static CallWideningDecision | decideCallWidening (VPInstruction &VPI, ArrayRef< VPValue * > Ops, ElementCount VF, VPCostContext &CostCtx) |
Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant. | |
Detailed Description
This file implements a set of utility VPlan to VPlan transformations.
Definition in file VPlanTransforms.cpp.
Typedef Documentation
◆ ExtendKind
Definition at line 3843 of file VPlanTransforms.cpp.
Function Documentation
◆ addReplicateRegions()
|
static |
Definition at line 641 of file VPlanTransforms.cpp.
References llvm::VPBlockUtils::blocksOnly(), createReplicateRegion(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getExiting(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::RegionBase< Tr >::getParent(), llvm::VPBlockBase::getParent(), llvm::Value::hasName(), llvm::VPBlockUtils::insertOnEdge(), llvm::make_isa_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPRegionBlock::setExiting(), llvm::VPBasicBlock::splitAt(), llvm::SplitBlock(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ areVFParamsOk()
|
static |
Returns true if Info's parameter kinds are compatible with Args.
Definition at line 6091 of file VPlanTransforms.cpp.
References llvm::all_of(), and llvm::PredicatedScalarEvolution::getSE().
Referenced by findVectorVariant().
◆ canFoldOuterExtendIntoMul()
|
static |
Returns true if the narrow Mul in ext(mul(ext(A), ext(B))) or ext(mul(ext(A), C)) cannot wrap.
The kind of the inner extends selects whether nuw (zext) or nsw (sext) is required on Mul.
Definition at line 3852 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::dyn_cast(), getPartialReductionExtendKind(), llvm::Type::getScalarSizeInBits(), LHS, Mul, and RHS.
Referenced by tryToMatchAndCreateMulAccumulateReduction().
◆ canHoistOrSinkWithNoAliasCheck()
|
static |
Check if a memory operation doesn't alias with memory operations using scoped noalias metadata, in blocks in the single-successor chain between FirstBB and LastBB.
If SinkInfo is std::nullopt, only recipes that may write to memory are checked (for load hoisting). Otherwise recipes that both read and write memory are checked, and SCEV is used to prove no-alias between the group leader and other replicate recipes (for store sinking).
Definition at line 284 of file VPlanTransforms.cpp.
Referenced by cannotHoistOrSinkRecipe(), canSinkStoreWithNoAliasCheck(), and llvm::VPlanTransforms::hoistPredicatedLoads().
◆ canNarrowLoad()
|
static |
Returns true if V is VPWidenLoadRecipe or VPInterleaveRecipe that can be converted to a narrower recipe.
V is used by a wide recipe that feeds a store interleave group at index Idx, WideMember0 is the recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to an index-independent load if it feeds all wide ops at all indices (OpV must be the operand at index OpIdx for both the recipe at lane 0, WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if V is defined at Idx of a load interleave group. A live-in or recipe defined outside the loop region can be converted, if it is the same across all lanes, or we can create a BuildVector for it.
Definition at line 4348 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getOperand(), llvm::VPValue::getScalarType(), IR, and llvm::VPValue::isDefinedOutsideLoopRegions().
Referenced by canNarrowOps().
◆ canNarrowOps()
|
static |
Returns true if the wide recipes in Ops and their operand trees can be narrowed.
FirstMembersOf maps each in-loop value to the first member list it was seen in, across all store groups.
Definition at line 4375 of file VPlanTransforms.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), canNarrowLoad(), canNarrowOps(), llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), llvm::equal(), llvm::vputils::getOpcode(), llvm::isa(), P, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by canNarrowOps(), and llvm::VPlanTransforms::narrowInterleaveGroups().
◆ cannotHoistOrSinkRecipe()
|
static |
Return true if we do not know how to (mechanically) hoist or sink a non-memory or memory recipe R out of a loop region.
When sinking, passing Sinking = true ensures that assumes aren't sunk.
Definition at line 2471 of file VPlanTransforms.cpp.
References canHoistOrSinkWithNoAliasCheck(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), llvm::vputils::getMemoryLocation(), llvm::isa(), llvm::PatternMatch::m_Intrinsic(), and llvm::SCEVPatternMatch::match().
Referenced by licm().
◆ canSinkStoreWithNoAliasCheck()
|
static |
Definition at line 4208 of file VPlanTransforms.cpp.
References llvm::ArrayRef< T >::back(), canHoistOrSinkWithNoAliasCheck(), llvm::ArrayRef< T >::front(), and llvm::vputils::getMemoryLocation().
Referenced by llvm::VPlanTransforms::sinkPredicatedStores().
◆ cloneBinOpForScalarIV()
|
static |
Create a scalar version of BinOp, with its WidenIV operand replaced by ScalarIV, and place it after ScalarIV's defining recipe.
Definition at line 4867 of file VPlanTransforms.cpp.
References assert(), llvm::VPWidenRecipe::clone(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPWidenRecipe::getOpcode(), llvm::VPRecipeBase::insertAfter(), and llvm::Instruction::isBinaryOp().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ collectComplementaryPredicatedMemOps()
|
static |
Definition at line 4095 of file VPlanTransforms.cpp.
References llvm::all_equal(), assert(), collectGroupedReplicateMemOps(), getLoadStoreValueType(), Groups, llvm::VPReplicateRecipe::isPredicated(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::map_range(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ collectGroupedReplicateMemOps()
|
static |
Collect either replicated Loads or Stores grouped by their address SCEV and their load-store type, in a deep-traversal of the vector loop region in Plan.
Definition at line 322 of file VPlanTransforms.cpp.
References A(), B(), llvm::VPBlockUtils::blocksOnly(), llvm::VPRegionBlock::getEntry(), getLoadStoreValueType(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getVectorLoopRegion(), Groups, llvm::isa(), llvm::make_isa_range(), llvm::VPDominatorTree::properlyDominates(), llvm::stable_sort(), llvm::to_vector(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::values(), and llvm::vp_depth_first_deep().
Referenced by collectComplementaryPredicatedMemOps().
◆ combineRecipe()
|
static |
Combine Def into a simpler recipe.
May modify or create new recipes via Builder.
Definition at line 1449 of file VPlanTransforms.cpp.
References llvm::all_equal(), llvm::all_of(), llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::Broadcast, C(), llvm::cast(), llvm::dyn_cast(), llvm::dyn_cast_if_present(), llvm::APInt::exactLogBase2(), llvm::CmpInst::FCMP_UNO, llvm::VPlanPatternMatch::findUserOf(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::CmpInst::getInversePredicate(), llvm::VPUser::getOperand(), llvm::Type::getScalarSizeInBits(), llvm::VPlan::getTrue(), llvm::VPDef::getVPSingleValue(), llvm::VPlan::getZero(), llvm::isa(), isAvailableAtEndOf(), llvm::APInt::isPowerOf2(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_Add(), llvm::MIPatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_Mul(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_False(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_LShr(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SelectLike(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificCmp(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_True(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_UDiv(), llvm::PatternMatch::m_URem(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPPhi(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), simplifyRecipe(), llvm::SmallVectorTemplateCommon< T, typename >::size(), swapSelectBranchWeights(), llvm::to_vector(), X, and Y.
Referenced by llvm::VPlanTransforms::combineRecipes().
◆ createReplicateRegion()
|
static |
Definition at line 581 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createReplicateRegion(), llvm::VPlan::createVPBasicBlock(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPReplicateRecipe::getMask(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::DebugLoc::getUnknown(), llvm::VPBlockUtils::insertTwoBlocksAfter(), llvm::VPReplicateRecipe::isSingleScalar(), llvm::VPReplicateRecipe::operandsWithoutMask(), llvm::VPValue::replaceAllUsesWith(), and llvm::to_vector().
Referenced by addReplicateRegions().
◆ decideCallWidening()
|
static |
Pick the cheapest widening for the call VPI at VF among scalarization, vector intrinsic, and vector library variant.
Definition at line 6153 of file VPlanTransforms.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::VPReplicateRecipe::computeCallCost(), llvm::VPWidenCallRecipe::computeCallCost(), llvm::VPWidenIntrinsicRecipe::computeCallCost(), findVectorVariant(), llvm::InstructionCost::getInvalid(), llvm::VPValue::getLiveInIRValue(), llvm::VPInstruction::getNumOperandsWithoutMask(), llvm::VPUser::getOperand(), llvm::VPRecipeValue::getScalarType(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::getVectorIntrinsicIDForCall(), IntrinsicCost, llvm::VPCostContext::isFreeScalarIntrinsic(), llvm::VPCostContext::isMaskRequired(), llvm::ElementCount::isScalar(), llvm::VPCostContext::L, llvm::VPCostContext::PSE, llvm::VPCostContext::TLI, and llvm::VPCostContext::willBeScalarized().
Referenced by llvm::VPlanTransforms::makeCallWideningDecisions().
◆ findRecipeWithMinAlign()
|
static |
Definition at line 4151 of file VPlanTransforms.cpp.
References A(), B(), llvm::cast(), and llvm::min_element().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ findVectorVariant()
|
static |
Find a vector variant of CI for VF, respecting MaskRequired.
Returns the variant function, or nullptr. Masked variants are assumed to take the mask as a trailing parameter.
Definition at line 6118 of file VPlanTransforms.cpp.
References areVFParamsOk(), llvm::find_if(), llvm::Module::getFunction(), llvm::VFDatabase::getMappings(), llvm::Instruction::getModule(), and llvm::CallBase::isNoBuiltin().
Referenced by decideCallWidening().
◆ getCommonMetadata()
|
static |
Definition at line 4083 of file VPlanTransforms.cpp.
References llvm::VPIRMetadata::clearExecutionFrequency(), llvm::drop_begin(), llvm::ArrayRef< T >::front(), and llvm::VPIRMetadata::intersect().
Referenced by llvm::VPlanTransforms::hoistPredicatedLoads(), and llvm::VPlanTransforms::sinkPredicatedStores().
◆ getExpressionIV()
|
static |
Check if V is a binary expression of a widened IV and a loop-invariant value.
Returns the widened IV if found, nullptr otherwise.
Definition at line 4848 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::VPValue::isDefinedOutsideLoopRegions(), llvm::Instruction::isIntDivRem(), and std::swap().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ getLoadStoreValueType()
|
static |
Get the value type of a VPInstruction or VPReplicateRecipe load or store.
Definition at line 312 of file VPlanTransforms.cpp.
References llvm::vputils::getOpcode().
Referenced by collectComplementaryPredicatedMemOps(), and collectGroupedReplicateMemOps().
◆ getMetadataOf()
|
static |
Returns the metadata attached to R, or an empty set for a recipe that does not carry any.
Definition at line 88 of file VPlanTransforms.cpp.
References llvm::dyn_cast().
Referenced by legalizeAndOptimizeInductions(), and narrowToSingleScalarRecipes().
◆ getOptimizableIVOf()
|
static |
Check if VPV is an untruncated wide induction, either before or after the increment.
If so return the header IV (before the increment), otherwise return null.
Definition at line 919 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPValue::getDefiningRecipe(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm_unreachable, llvm::VPlanPatternMatch::m_Binary(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_FAdd(), llvm::VPlanPatternMatch::m_GetElementPtr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::narrowInductionTruncates(), optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().
◆ getPartialReductionExtendKind()
|
static |
Definition at line 3845 of file VPlanTransforms.cpp.
References llvm::VPWidenCastRecipe::getOpcode(), and llvm::TargetTransformInfo::getPartialReductionExtendKind().
Referenced by canFoldOuterExtendIntoMul().
◆ getPredicatedThenBlock()
|
static |
If R is a triangle region, return the 'then' block of the triangle.
Definition at line 451 of file VPlanTransforms.cpp.
References llvm::cast(), and llvm::dyn_cast().
Referenced by mergeReplicateRegionsIntoSuccessors().
◆ getRecipesForUncountableExit()
|
static |
Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depends on can be traced back to live-ins and the addresses (in GEP/PtrAdd form) of any (non-masked) load used in generating the values for the comparison.
The recipes are stored in Recipes.
Definition at line 3205 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPValue::getDefiningRecipe(), llvm::VPUser::getNumOperands(), llvm::VPBasicBlock::getTerminator(), llvm::VPlanPatternMatch::m_AnyOf(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Freeze(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::PatternMatch::match_fn(), llvm::none_of(), llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by handleUncountableExitsWithSideEffects().
◆ getStepDirection()
|
static |
If S is an affine AddRec, returns true if its step is known to be positive and false if it is known to be negative.
Returns std::nullopt if S is not an affine AddRec, or if the sign of its step cannot be determined.
Definition at line 4886 of file VPlanTransforms.cpp.
References llvm::ScalarEvolution::isKnownNegative(), llvm::ScalarEvolution::isKnownPositive(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_AffineAddRec(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeFindIVReductions().
◆ getUnmaskedDivRemOpcode()
|
static |
Definition at line 1863 of file VPlanTransforms.cpp.
Referenced by narrowToSingleScalarRecipes().
◆ handleUncountableExitsWithSideEffects()
|
static |
Update Plan to mask memory operations in the loop based on whether the early exit is taken or not.
We're currently expecting to find a loop with properties similar to the following:
for.body: ir<indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0> EMIT ir<arrayidx> = getelementptr inbounds nuw ir< >, ir<indvars.iv> EMIT-SCALAR ir<%0> = load ir<arrayidx> EMIT ir<cmp1> = icmp sgt ir<%0>, ir<5> EMIT vp<%1> = masked-cond ir<cmp1> Successor(s): if.end
if.end: EMIT ir<arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<indvars.iv> EMIT-SCALAR ir<%2> = load ir<arrayidx3> EMIT ir<add> = add nsw ir<%2>, ir<42> EMIT ir<arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<indvars.iv> EMIT store ir<add>, ir<arrayidx5> EMIT ir<indvars.iv.next> = add nuw nsw ir<indvars.iv>, ir<1> EMIT vp<freeze> = freeze ir<%1> EMIT vp<%3> = any-of ir<freeze> EMIT ir<exitcond.not> = icmp eq ir<indvars.iv.next>, ir<10000> EMIT branch-on-two-conds vp<%3>, ir<exitcond.not> Successor(s): middle.block, middle.block, for.body
We currently expect LoopVectorizationLegality to ensure that:
- There must also be a counted exit. We will need to support speculative or first-faulting loads before we can remove this restriction.
- Any stores within the loop must not alias with the load used for the uncountable exit. We can relax this a bit with runtime aliasing checks.
- Other memory operations in the loop can take place before or after the uncountable exit, but must also be unconditional. We need to support combining the conditions in VPlanPredicator.
- The loop must have a single unconditional load contributing to the uncountable exit comparison, and the other term must be loop-invariant. Improving upon this requires work in getRecipesForUncountableExit to handle more complex recipe graphs.
Definition at line 3355 of file VPlanTransforms.cpp.
References _, llvm::VPInstruction::ActiveLaneMask, assert(), llvm::cast(), Cond, llvm::VPBuilderBase< InserterTy >::createAdd(), llvm::VPBuilderBase< InserterTy >::createFirstActiveLane(), llvm::VPBuilderBase< InserterTy >::createICmp(), llvm::VPBuilderBase< InserterTy >::createNaryOp(), llvm::VPBuilderBase< InserterTy >::createScalarZExtOrTrunc(), llvm::dwarf_linker::DebugLoc, llvm::VPBlockUtils::disconnectBlocks(), DL, llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::VPBasicBlock::end(), llvm::erase(), llvm::VPInstruction::ExtractLane, llvm::find_singleton(), llvm::VPBasicBlock::front(), llvm::ScalarEvolution::getConstant(), llvm::VPlan::getDataLayout(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPUser::getOperand(), getRecipesForUncountableExit(), llvm::VPlan::getScalarPreheader(), llvm::VPRecipeValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::VPlan::getZero(), I, llvm::CmpInst::ICMP_EQ, llvm::is_contained(), llvm::isDereferenceableAndAlignedInLoop(), IV, llvm::Load, llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::range_size(), llvm::LoopVectorizationUtils::reportVectorizationFailure(), llvm::reverse(), llvm::VPUser::setOperand(), and llvm::vp_rpo_plain_cfg_loop_body().
Referenced by llvm::VPlanTransforms::handleUncountableEarlyExits().
◆ isAlreadyNarrow()
Returns true if VPValue is a narrow VPValue.
Definition at line 4471 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), and llvm::isa().
Referenced by narrowInterleaveGroupOp(), and llvm::VPlanTransforms::narrowInterleaveGroups().
◆ isAvailableAtEndOf()
|
static |
Returns true if V is available at the end of VPBB, i.e.
it either is a live-in from the original IR or defined in VPBB.
Definition at line 1425 of file VPlanTransforms.cpp.
References llvm::VPRecipeBase::getParent(), and llvm::isa().
Referenced by combineRecipe().
◆ isConditionTrueViaVFAndUF()
|
static |
Return true if Cond is known to be true for given BestVF and BestUF.
Definition at line 2148 of file VPlanTransforms.cpp.
References llvm::any_of(), assert(), C(), Cond, llvm::VPRegionBlock::getCanonicalIV(), llvm::ScalarEvolution::getElementCount(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::isa(), llvm::ScalarEvolution::isKnownPredicate(), llvm::VPlanPatternMatch::m_BinaryOr(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by simplifyBranchConditionForVFAndUF().
◆ isConsecutiveInterleaveGroup()
|
static |
Returns VF from VFs if IR is a full interleave group with factor and number of members both equal to VF.
The interleave group must also access the full vector width.
Definition at line 4427 of file VPlanTransforms.cpp.
References llvm::all_of(), assert(), llvm::VPDef::definedValues(), llvm::VPInterleaveBase::getInterleaveGroup(), llvm::VPInterleaveBase::getMask(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getScalarType(), llvm::VPInterleaveBase::getStoredValues(), llvm::VPDef::getVPValue(), llvm::TargetTransformInfo::RGK_FixedWidthVector, llvm::TargetTransformInfo::RGK_ScalableVector, and Size.
Referenced by llvm::VPlanTransforms::narrowInterleaveGroups().
◆ legalizeAndOptimizeInductions()
|
static |
Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as VPWidenPointerInductionRecipe will generate vectors only.
If some users require vectors while other require scalars, the scalar uses need to extract the scalars from the generated vectors (Note that this is different to how int/fp inductions are handled). Legalize extract-from-ends using uniform VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so the correct end value is available. Also optimize VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by providing them scalar steps built on the canonical scalar IV and update the original IV's users. This is an optional optimization to reduce the needs of vector extracts.
Definition at line 820 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), llvm::vputils::collectUsersRecursively(), llvm::vputils::createScalarIVSteps(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), getMetadataOf(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalableVF(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::APInt::isNonNegative(), llvm::vputils::isSingleScalar(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Binary(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_isa_range(), llvm::PatternMatch::match(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reverse(), llvm::vputils::scalarizeVPWidenPointerInduction(), Users, and llvm::VPIRFlags::WrapFlagsTy::withoutNoSignedWrap().
Referenced by llvm::VPlanTransforms::optimize().
◆ licm()
|
static |
Move loop-invariant recipes out of the vector loop region in Plan.
Definition at line 2492 of file VPlanTransforms.cpp.
References llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), cannotHoistOrSinkRecipe(), llvm::cast(), llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::dyn_cast(), llvm::VPBasicBlock::end(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::make_early_inc_range(), llvm::VPDominatorTree::properlyDominates(), llvm::reverse(), and llvm::vp_depth_first_shallow().
◆ loopMayHaveSideEffects()
|
static |
Definition at line 3532 of file VPlanTransforms.cpp.
References llvm::any_of(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::PatternMatch::match(), and llvm::vp_rpo_plain_cfg_loop_body().
Referenced by llvm::VPlanTransforms::handleUncountableEarlyExits().
◆ m_Countable()
|
static |
Matches an exit condition formed by comparing the current value of a affine add recurrence in the given loop with a stride of 1 against a loop-invariant term.
Binds the comparison for the condition.
Definition at line 3123 of file VPlanTransforms.cpp.
Referenced by llvm::VPlanTransforms::splitCombinedExits().
◆ m_Uncountable()
|
static |
Matches an exit condition formed by comparing a value loaded from memory with a loop-invariant term.
Binds the comparison for the condition.
Definition at line 3089 of file VPlanTransforms.cpp.
References Cond, llvm::PatternMatch::m_c_Cmp(), llvm::VPlanPatternMatch::m_LiveIn(), llvm::VPlanPatternMatch::m_VPInstruction(), and llvm::VPlanPatternMatch::m_VPValue().
Referenced by llvm::VPlanTransforms::splitCombinedExits().
◆ mergeReplicateRegionsIntoSuccessors()
Definition at line 473 of file VPlanTransforms.cpp.
References assert(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPIRMetadata::clearExecutionFrequency(), llvm::VPBlockUtils::connectBlocks(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBlockUtils::disconnectBlocks(), llvm::dyn_cast_or_null(), llvm::SmallPtrSetImplBase::empty(), llvm::VPlan::getContext(), llvm::VPlan::getEntry(), llvm::VPIRMetadata::getExecutionFrequency(), llvm::VPBasicBlock::getFirstNonPhi(), getPredicatedThenBlock(), llvm::VPBlockBase::getSingleSuccessor(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::VPValue::replaceUsesWithIf(), llvm::reverse(), llvm::VPIRMetadata::setExecutionFrequency(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ narrowInterleaveGroupOp()
|
static |
Definition at line 4482 of file VPlanTransforms.cpp.
References llvm::all_of(), llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPInstruction::BuildVector, llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBuilderBase< InserterTy >::createWidenLoad(), llvm::ArrayRef< T >::drop_front(), llvm::dyn_cast(), E(), llvm::ArrayRef< T >::front(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), isAlreadyNarrow(), N, narrowInterleaveGroupOp(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by narrowInterleaveGroupOp().
◆ narrowToSingleScalarRecipes()
|
static |
Definition at line 1878 of file VPlanTransforms.cpp.
References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastLane, llvm::VPInstruction::ExtractLastPart, llvm::VPInstruction::ExtractPenultimateElement, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPlan::getEntry(), getMetadataOf(), llvm::vputils::getOpcode(), llvm::DebugLoc::getUnknown(), getUnmaskedDivRemOpcode(), llvm::VPlan::hasScalarVFOnly(), llvm::is_contained(), llvm::isa(), llvm::vputils::isDeadRecipe(), llvm::vputils::isSingleScalar(), llvm::vputils::isUniformAcrossVFsAndUFs(), llvm::make_early_inc_range(), llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), Opc, llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::optimize().
◆ optimizeEarlyExitInductionUser()
|
static |
Attempts to optimize the induction variable exit values for users in the early exit block.
Definition at line 976 of file VPlanTransforms.cpp.
References B(), llvm::cast(), DL, llvm::dyn_cast_or_null(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPRegionBlock::getCanonicalIVType(), llvm::VPlan::getConstantInt(), getOptimizableIVOf(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_FirstActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeLatchExitInductionUser()
|
static |
Attempts to optimize the induction variable exit values for users in the exit block coming from the latch in the original scalar loop.
Definition at line 1056 of file VPlanTransforms.cpp.
References assert(), B(), llvm::cast(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), getOptimizableIVOf(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::DebugLoc::getUnknown(), llvm::VPlan::getZero(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm_unreachable, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeLatchExitIVUserViaSCEV()
|
static |
Definition at line 1103 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), DL, llvm::VPSCEVExpander::expand(), llvm::VPlan::getConstantInt(), llvm::VPValue::getScalarType(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::DebugLoc::getUnknown(), llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::Type::isIntOrPtrTy(), llvm::Type::isPointerTy(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLaneOfLastPart(), llvm::VPlanPatternMatch::m_HeaderMask(), llvm::VPlanPatternMatch::m_LastActiveLane(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_AffineAddRec(), llvm::SCEVPatternMatch::m_SpecificLoop(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ optimizeVectorInductionWidthForTCAndVFUF()
|
static |
Optimize the width of vector induction variables in Plan based on a known constant Trip Count, BestVF and BestUF.
Definition at line 2074 of file VPlanTransforms.cpp.
References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::cast(), llvm::IntegerType::get(), llvm::APInt::getActiveBits(), llvm::VPlan::getBackedgeTakenCount(), llvm::APInt::getBitWidth(), llvm::VPlan::getConstantInt(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPWidenInductionRecipe::getPHINode(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getSingleUser(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::VPWidenInductionRecipe::getVFValue(), llvm::VPlan::getZero(), llvm::VPValue::hasMoreThanOneUniqueUser(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::VPWidenIntOrFpInductionRecipe::isTruncated(), llvm::PatternMatch::m_APInt(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::PowerOf2Ceil(), llvm::APIntOps::RoundingUDiv(), uint64_t, and llvm::APInt::UP.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ reassociateHeaderMask()
|
static |
Reassociate (headermask && x) && y -> headermask && (x && y) to allow the header mask to be simplified further when tail folding, e.g.
in optimizeEVLMasks.
Definition at line 1838 of file VPlanTransforms.cpp.
References llvm::append_range(), llvm::cast(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPRegionBlock::getHeaderMask(), llvm::VPlan::getVectorLoopRegion(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::VPValue::users(), users, X, and Y.
Referenced by llvm::VPlanTransforms::optimize().
◆ removeCommonBlendMask()
|
static |
Try to see if all of Blend's masks share a common value logically and'ed and remove it from the masks.
Definition at line 1979 of file VPlanTransforms.cpp.
References llvm::VPValue::getDefiningRecipe(), llvm::VPBlendRecipe::getMask(), llvm::VPBlendRecipe::getNumIncomingValues(), llvm::VPUser::getOperand(), I, llvm::VPBlendRecipe::isNormalized(), llvm::PatternMatch::m_LogicalAnd(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPBlendRecipe::setMask().
Referenced by simplifyBlends().
◆ removeRedundantInductionCasts()
|
static |
Remove redundant casts of inductions.
Such redundant casts are casts of induction variables that can be ignored, because we already proved that the casted phi is equal to the uncasted phi in the vectorized loop. There is no need to vectorize the cast - the same value can be used for both the phi and casts in the vector loop.
Definition at line 717 of file VPlanTransforms.cpp.
References llvm::dyn_cast(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), IV, llvm::make_isa_range(), llvm::VPBasicBlock::phis(), llvm::VPValue::replaceAllUsesWith(), llvm::reverse(), and llvm::VPValue::users().
Referenced by llvm::VPlanTransforms::optimize().
◆ replaceMaskWithCompareForScalarPlan()
|
static |
Definition at line 2183 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::VPlan::getConstantInt(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), llvm::CmpInst::ICMP_ULT, llvm::ElementCount::isScalar(), llvm::PatternMatch::m_ConstantInt(), llvm::VPlanPatternMatch::m_ExtractVectorForPart(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_WideActiveLaneMask(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), and uint64_t.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ simplifyBlends()
|
static |
Normalize and simplify VPBlendRecipes.
Should be run after combineRecipes to make sure the masks are simplified.
Simplify BLEND a, b, Not(mask) -> BLEND b, a, mask.
Definition at line 1996 of file VPlanTransforms.cpp.
References llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::cast_or_null(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPlanPatternMatch::m_False(), llvm::MIPatternMatch::m_Not(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::make_isa_range(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::vputils::recursivelyDeleteDeadRecipes(), removeCommonBlendMask(), llvm::VPValue::replaceAllUsesWith(), llvm::SmallPtrSetImplBase::size(), llvm::VPValue::user_empty(), and llvm::vp_depth_first_shallow().
Referenced by llvm::VPlanTransforms::optimize().
◆ simplifyBranchConditionForVFAndUF()
|
static |
Try to simplify the branch condition of Plan.
This may restrict the resulting plan to BestVF and BestUF.
Definition at line 2223 of file VPlanTransforms.cpp.
References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::VPBasicBlock::back(), llvm::VPInstruction::BranchOnCond, C(), Cond, llvm::ScalarEvolution::getElementCount(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_ULE, llvm::isa(), isConditionTrueViaVFAndUF(), llvm::ScalarEvolution::isKnownPredicate(), llvm::PatternMatch::m_Add(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_BranchOnTwoConds(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ExtractVectorForPart(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_WideActiveLaneMask(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), and llvm::Offset.
Referenced by llvm::VPlanTransforms::optimizeForVFAndUF().
◆ simplifyLogicalRecipe()
|
static |
Try to simplify logical and bitwise recipes in Def.
Definition at line 1210 of file VPlanTransforms.cpp.
References assert(), llvm::VPlan::getAllOnesValue(), llvm::VPlan::getFalse(), llvm::VPlan::getZero(), llvm::CmpInst::ICMP_NE, llvm::MIPatternMatch::m_AllOnes(), llvm::VPlanPatternMatch::m_c_BinaryAnd(), llvm::VPlanPatternMatch::m_c_BinaryOr(), llvm::PatternMatch::m_c_LogicalAnd(), llvm::PatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_False(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_SpecificICmp(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), and X.
Referenced by simplifyRecipe().
◆ simplifyRecipe()
|
static |
Return an existing value or a live in for VPSingleDefRecipe Def if possible.
This shouldn't create or modify recipes.
Definition at line 1279 of file VPlanTransforms.cpp.
References A(), llvm::all_equal(), B(), llvm::cast(), DL, llvm::drop_begin(), llvm::dyn_cast(), llvm::VPRegionBlock::getCanonicalIV(), llvm::VPlan::getConcreteUF(), llvm::VPlan::getDataLayout(), llvm::VPRecipeBase::getRegion(), llvm::VPlan::getZero(), llvm::VPlan::hasScalarVFOnly(), llvm::isa(), llvm::vputils::isSingleScalar(), llvm::VPlan::isUnrolled(), llvm::PatternMatch::m_AShr(), llvm::PatternMatch::m_BitCast(), llvm::VPlanPatternMatch::m_Broadcast(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_Mul(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatchHelpers::m_CombineOr(), llvm::VPlanPatternMatch::m_ComputeReductionResult(), llvm::PatternMatch::m_ConstantInt(), llvm::VPlanPatternMatch::m_DerivedIV(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_ExtractLane(), llvm::VPlanPatternMatch::m_ExtractLastLane(), llvm::VPlanPatternMatch::m_ExtractLastPart(), llvm::VPlanPatternMatch::m_ExtractPenultimateElement(), llvm::PatternMatch::m_LShr(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Trunc(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPIRValue(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::SCEVPatternMatch::match(), llvm::vputils::onlyFirstLaneUsed(), simplifyLogicalRecipe(), llvm::vputils::tryToFoldLiveIns(), and uint64_t.
Referenced by combineRecipe().
◆ sinkScalarOperands()
Definition at line 356 of file VPlanTransforms.cpp.
References llvm::all_of(), llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::vputils::cannotHoistOrSinkRecipe(), llvm::cast(), Changed, llvm::VPSingleDefRecipe::clone(), llvm::VPBuilderBase<>::createSingleScalarOp(), llvm::vputils::doesGeneratePerAllLanes(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPRecipeValue::getScalarType(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPSingleDefRecipe::getUnderlyingInstr(), llvm::VPlan::hasScalarVFOnly(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::VPRecipeBase::insertBefore(), llvm::isa_and_nonnull(), llvm::vputils::isSingleScalar(), llvm::make_filter_range(), llvm::VPRecipeBase::moveBefore(), llvm::VPUser::operands(), llvm::VPValue::replaceUsesWithIf(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::VPValue::users(), and llvm::vp_depth_first_deep().
Referenced by llvm::VPlanTransforms::createAndOptimizeReplicateRegions().
◆ swapSelectBranchWeights()
|
static |
Swap the branch weights recorded for R, a select whose two selected operands are being swapped,.
Definition at line 1264 of file VPlanTransforms.cpp.
References assert(), llvm::MDBuilder::createBranchWeights(), llvm::dyn_cast(), llvm::extractBranchWeights(), llvm::VPlan::getContext(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by combineRecipe().
◆ tryToComputeEndValueForInduction()
|
static |
Compute the end value for WideIV, unless it is truncated.
Creates a VPDerivedIVRecipe for non-canonical inductions.
Definition at line 1022 of file VPlanTransforms.cpp.
References llvm::VPBuilderBase< InserterTy >::createDerivedIV(), llvm::VPBuilderBase< InserterTy >::createScalarCast(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), and llvm::PatternMatch::match().
Referenced by llvm::VPlanTransforms::optimizeInductionLiveOutUsers().
◆ tryToCreateAbstractReductionRecipe()
|
static |
This function tries to create abstract recipes from the reduction recipe for following optimizations and cost estimation.
Definition at line 4048 of file VPlanTransforms.cpp.
References llvm::VPBlockBase::getParent(), llvm::VPRecipeBase::insertBefore(), Range, tryToMatchAndCreateExtendedReduction(), and tryToMatchAndCreateMulAccumulateReduction().
Referenced by llvm::VPlanTransforms::convertToAbstractRecipes().
◆ tryToMatchAndCreateExtendedReduction()
|
static |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
The created recipe must be decomposed to its constituent recipes before execution.
Definition at line 3800 of file VPlanTransforms.cpp.
References A(), assert(), llvm::cast(), CostKind, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::InstructionCost::getInvalid(), llvm::RecurrenceDescriptor::getOpcode(), llvm::Type::getScalarType(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatchHelpers::m_Isa(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Range, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().
Referenced by tryToCreateAbstractReductionRecipe().
◆ tryToMatchAndCreateMulAccumulateReduction()
|
static |
This function tries convert extended in-loop reductions to VPExpressionRecipe and clamp the Range if it is beneficial and valid.
The created VPExpressionRecipe must be decomposed to its constituent recipes before execution. Patterns of the VPExpressionRecipe: reduce.add(mul(...)), reduce.add(mul(ext(A), ext(B))), reduce.add(ext(mul(ext(A), ext(B)))). reduce.fadd(fmul(ext(A), ext(B)))
Definition at line 3877 of file VPlanTransforms.cpp.
References A(), B(), llvm::canConstantBeExtended(), canFoldOuterExtendIntoMul(), llvm::cast(), CostKind, llvm::dyn_cast(), llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPValue::getDefiningRecipe(), llvm::RecurrenceDescriptor::getOpcode(), llvm::VPWidenCastRecipe::getOpcode(), llvm::VPUser::getOperand(), llvm::TargetTransformInfo::getPartialReductionExtendKind(), llvm::Type::getScalarType(), llvm::VPRecipeValue::getScalarType(), llvm::VPValue::getScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::Type::isFloatingPointTy(), llvm::InstructionCost::isValid(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Sub(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), Mul, Range, llvm::Sub, llvm::TargetTransformInfo::TCK_RecipThroughput, and llvm::toVectorTy().
Referenced by tryToCreateAbstractReductionRecipe().
◆ tryToRemoveDeadCycle()
|
static |
If R is a phi-like recipe starting a dead cycle of recipes, erase all reachable recipes of the dead cycle and return true.
Otherwise leave the plan unchanged and return false.
Definition at line 756 of file VPlanTransforms.cpp.
References llvm::cast(), llvm::vputils::collectUsersRecursively(), llvm::dyn_cast(), llvm::isa(), and llvm::vputils::recursivelyDeleteDeadRecipes().
Referenced by llvm::VPlanTransforms::removeDeadRecipes().
◆ updatePartialReductionPhiAndResult()
|
static |
Definition at line 5652 of file VPlanTransforms.cpp.
References assert(), llvm::cast(), llvm::vputils::findComputeReductionResult(), llvm::FSub, llvm::VPlan::getConstantInt(), llvm::VPIRFlags::getDefaultFlags(), llvm::VPBuilderBase<>::getToInsertAfter(), llvm::not_equal_to(), llvm::VPInstruction::ReductionStartVector, llvm::VPValue::replaceUsesWithIf(), and llvm::Sub.
Referenced by llvm::VPlanTransforms::createPartialReductions().
Generated on for LLVM by
1.14.0