22#define DEBUG_TYPE "loop-vectorize"
30 auto RecipeI = VPBB->
begin();
32 unsigned NumActiveLaneMaskPhiRecipes = 0;
34 bool IsHeaderVPBB = ParentR && !ParentR->
isReplicator() &&
36 while (RecipeI !=
End && RecipeI->isPhi()) {
37 if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
38 NumActiveLaneMaskPhiRecipes++;
40 if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI)) {
41 errs() <<
"Found non-header PHI recipe in header VPBB";
42#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
49 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
50 errs() <<
"Found header PHI recipe in non-header VPBB";
51#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
61 if (NumActiveLaneMaskPhiRecipes > 1) {
62 errs() <<
"There should be no more than one VPActiveLaneMaskPHIRecipe";
66 while (RecipeI !=
End) {
67 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
68 errs() <<
"Found phi-like recipe after non-phi recipe";
70#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
74 std::prev(RecipeI)->dump();
93 RecipeNumbering[&R] = Cnt++;
102 auto CheckEVLRecipiesInsts = [&](
const VPRecipeBase *R) {
103 if (isa<VPEVLBasedIVPHIRecipe>(R)) {
105 errs() <<
"EVL PHI recipe not in entry block!\n";
108 if (!EVLFound.
insert(VPDef::VPEVLBasedIVPHISC).second) {
109 errs() <<
"EVL PHI recipe inserted more than once!\n";
114 if (
const auto *RInst = dyn_cast<VPInstruction>(R);
117 errs() <<
"EVL instruction not in the header block!\n";
120 if (!EVLFound.
insert(RInst->getOpcode() + VPDef::VPLastPHISC).second) {
121 errs() <<
"EVL instruction inserted more than once!\n";
124 if (VPWidenMemRecipe) {
125 errs() <<
"Use of EVL instruction by widen memory recipe before "
131 if (isa<VPWidenMemoryRecipe>(R))
132 VPWidenMemRecipe = R;
137 if (!CheckEVLRecipiesInsts(&R))
139 for (
const VPValue *V : R.definedValues()) {
140 for (
const VPUser *U : V->users()) {
141 auto *UI = dyn_cast<VPRecipeBase>(U);
144 isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI))
149 if (UI->getParent() == VPBB) {
150 if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
151 errs() <<
"Use before def!\n";
158 errs() <<
"Use before def!\n";
171 for (
const auto *
Block : VPBlockVec) {
180 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
183 (VPBB && VPBB->getParent() && VPBB->isExiting() &&
184 !VPBB->getParent()->isReplicator())) {
185 if (!VPBB || !VPBB->getTerminator()) {
186 errs() <<
"Block has multiple successors but doesn't "
187 "have a proper branch recipe!\n";
191 if (VPBB && VPBB->getTerminator()) {
192 errs() <<
"Unexpected branch recipe!\n";
202 errs() <<
"Multiple instances of the same successor.\n";
208 const auto &SuccPreds = Succ->getPredecessors();
210 errs() <<
"Missing predecessor link.\n";
221 errs() <<
"Multiple instances of the same predecessor.\n";
227 if (Pred->getParent() != VPB->
getParent()) {
228 errs() <<
"Predecessor is not in the same region.\n";
233 const auto &PredSuccs = Pred->getSuccessors();
235 errs() <<
"Missing successor link.\n";
249 if (VPB->getParent() !=
Region) {
250 errs() <<
"VPBlockBase has wrong parent\n";
268 if (Entry->getNumPredecessors() != 0) {
269 errs() <<
"region entry block has predecessors\n";
273 errs() <<
"region exiting block has successors\n";
288 const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB);
289 return !SubRegion || verifyRegionRec(SubRegion, VPDT);
299 [&VPDT](
const VPBlockBase *VPB) { return !verifyBlock(VPB, VPDT); }))
307 errs() <<
"VPlan Top Region should have no parent.\n";
313 errs() <<
"VPlan entry block is not a VPBasicBlock\n";
317 if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
318 errs() <<
"VPlan vector loop header does not start with a "
319 "VPCanonicalIVPHIRecipe\n";
325 errs() <<
"VPlan exiting block is not a VPBasicBlock\n";
329 if (Exiting->
empty()) {
330 errs() <<
"VPlan vector loop exiting block must end with BranchOnCount or "
331 "BranchOnCond VPInstruction but is empty\n";
335 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->
end()));
338 errs() <<
"VPlan vector loop exit must end with BranchOnCount or "
339 "BranchOnCond VPInstruction\n";
344 if (KV.second->getNumOperands() != 1) {
345 errs() <<
"live outs must have a single operand\n";
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static bool verifyPhiRecipes(const VPBasicBlock *VPBB)
static bool verifyRegion(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static bool verifyRegionRec(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static bool verifyVPBasicBlock(const VPBasicBlock *VPBB, const VPDominatorTree &VPDT)
static bool verifyBlock(const VPBlockBase *VPB, const VPDominatorTree &VPDT)
static bool hasDuplicates(const SmallVectorImpl< VPBlockBase * > &VPBlockVec)
Utility function that checks whether VPBlockVec has duplicate VPBlockBases.
static bool verifyBlocksInRegion(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Helper function that verifies the CFG invariants of the VPBlockBases within Region.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Implements a dense probed hash-table based set.
Core dominator tree base class.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Implements a dense probed hash-table based set with some number of buckets stored inline.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
iterator begin()
Recipe iterator methods.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBlocksTy & getPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
const VPBlocksTy & getSuccessors() const
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
const VPBlockBase * getExiting() const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.