Go to the documentation of this file.
25 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
53 class InnerLoopVectorizer;
56 class RecurrenceDescriptor;
80 "Both Start and End should have the same scalable flag");
82 "Expected Start to be a power of 2");
160 return Instance.Part <
I->second.size() &&
177 "need to overwrite existing value");
178 Iter->second[Part] = V;
184 auto &PerPartVec = Iter.first->second;
185 while (PerPartVec.size() <=
Instance.Part)
186 PerPartVec.emplace_back();
187 auto &Scalars = PerPartVec[
Instance.Part];
188 while (Scalars.size() <=
Instance.Lane)
189 Scalars.push_back(
nullptr);
190 assert(!Scalars[
Instance.Lane] &&
"should overwrite existing value");
198 "need to overwrite existing value");
200 "need to overwrite existing value");
202 "need to overwrite existing value");
260 const unsigned char SubclassID;
287 VPlan *Plan =
nullptr;
297 assert(Predecessor &&
"Cannot add nullptr predecessor!");
298 Predecessors.push_back(Predecessor);
303 auto Pos =
find(Predecessors, Predecessor);
304 assert(Pos &&
"Predecessor does not exist");
305 Predecessors.
erase(Pos);
311 assert(Pos &&
"Successor does not exist");
312 Successors.
erase(Pos);
324 using VPBlockTy =
enum { VPBasicBlockSC, VPRegionBlockSC };
373 return (Successors.size() == 1 ? *Successors.begin() :
nullptr);
379 return (Predecessors.size() == 1 ? *Predecessors.begin() :
nullptr);
446 assert(Successors.empty() &&
"Setting one successor when others exist.");
456 assert(Successors.empty() &&
"Setting two successors when others exist.");
457 assert(Condition &&
"Setting two successors without condition!");
459 appendSuccessor(IfTrue);
460 appendSuccessor(IfFalse);
467 assert(Predecessors.empty() &&
"Block predecessors already set.");
468 for (
auto *Pred : NewPreds)
469 appendPredecessor(Pred);
529 template <
typename IterT>
571 return cast<Instruction>(
getVPValue()->getUnderlyingValue());
574 return cast<Instruction>(
getVPValue()->getUnderlyingValue());
585 return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
586 Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
587 Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
588 Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
589 Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
590 Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
591 Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
592 Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
593 Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
594 Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
595 Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
608 Not = Instruction::OtherOpsEnd + 1,
616 typedef unsigned char OpcodeTy;
653 return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
675 Opcode == Instruction::Invoke || Opcode ==
SLPStore;
683 case Instruction::Br:
685 case Instruction::Switch:
686 case Instruction::IndirectBr:
687 case Instruction::Resume:
688 case Instruction::CatchRet:
689 case Instruction::Unreachable:
690 case Instruction::Fence:
691 case Instruction::AtomicRMW:
704 template <
typename IterT>
713 return D->getVPDefID() == VPRecipeBase::VPWidenSC;
731 template <
typename IterT>
740 return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
758 template <
typename IterT>
763 InvariantCond(InvariantCond) {}
769 return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
782 bool IsPtrLoopInvariant;
786 template <
typename IterT>
790 IsIndexLoopInvariant(
GEP->getNumIndices(),
false) {}
792 template <
typename IterT>
797 IsIndexLoopInvariant(
GEP->getNumIndices(),
false) {
800 IsIndexLoopInvariant[
Index.index()] =
807 return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
826 :
VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) {
839 return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
863 return dyn_cast_or_null<TruncInst>(
getVPValue(0)->getUnderlyingValue());
866 return dyn_cast_or_null<TruncInst>(
getVPValue(0)->getUnderlyingValue());
887 this->RdxDesc = &RdxDesc;
899 return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
920 IncomingBlocks.push_back(IncomingBlock);
944 "Expected either a single incoming value or a positive even number "
950 return D->getVPDefID() == VPRecipeBase::VPBlendSC;
978 bool HasMask =
false;
986 if (
I->getType()->isVoidTy())
991 for (
auto *SV : StoredValues)
1002 return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
1063 return D->getVPDefID() == VPRecipeBase::VPReductionSC;
1098 template <
typename IterT>
1100 bool IsUniform,
bool IsPredicated =
false)
1102 IsUniform(IsUniform), IsPredicated(IsPredicated) {
1108 AlsoPack = IsPredicated && !
I->use_empty();
1115 return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
1149 return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
1159 O <<
" +\n" << Indent <<
"\"BRANCH-ON-MASK ";
1192 return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
1218 bool isMasked()
const {
1238 return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
1254 bool isStore()
const {
return isa<StoreInst>(Ingredient); }
1258 assert(
isStore() &&
"Stored value only available for store instructions");
1281 return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1333 inline size_t size()
const {
return Recipes.size(); }
1334 inline bool empty()
const {
return Recipes.empty(); }
1345 return &VPBasicBlock::Recipes;
1350 return V->
getVPBlockID() == VPBlockBase::VPBasicBlockSC;
1354 assert(Recipe &&
"No recipe to append.");
1355 assert(!Recipe->Parent &&
"Recipe already in VPlan");
1356 Recipe->Parent =
this;
1357 Recipes.
insert(InsertPt, Recipe);
1400 const std::string &Name =
"",
bool IsReplicator =
false)
1401 :
VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
1402 IsReplicator(IsReplicator) {
1403 assert(Entry->getPredecessors().empty() &&
"Entry block has predecessors.");
1405 Entry->setParent(
this);
1409 :
VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exit(nullptr),
1410 IsReplicator(IsReplicator) {}
1415 Entry->dropAllReferences(&DummyValue);
1422 return V->
getVPBlockID() == VPBlockBase::VPRegionBlockSC;
1432 "Entry block cannot have predecessors.");
1450 "Exit block cannot have successors.");
1483 return N->getSuccessors().begin();
1487 return N->getSuccessors().end();
1498 return N->getSuccessors().begin();
1502 return N->getSuccessors().end();
1515 return N->getPredecessors().begin();
1519 return N->getPredecessors().end();
1574 return N.Graph->getExit();
1615 VPValue *BackedgeTakenCount =
nullptr;
1631 Entry->setPlan(
this);
1638 Block->dropAllReferences(&DummyValue);
1642 for (
VPValue *VPV : VPValuesToFree)
1644 if (BackedgeTakenCount)
1645 delete BackedgeTakenCount;
1658 Block->setPlan(
this);
1664 if (!BackedgeTakenCount)
1665 BackedgeTakenCount =
new VPValue();
1666 return BackedgeTakenCount;
1680 VPExternalDefs.
insert(VPVal);
1684 assert(V &&
"Trying to add a null Value to VPlan");
1685 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
1687 Value2VPValue[V] = VPV;
1688 VPValuesToFree.push_back(VPV);
1692 assert(V &&
"Trying to add a null Value to VPlan");
1693 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
1694 Value2VPValue[V] = VPV;
1698 assert(V &&
"Trying to get the VPValue of a null Value");
1699 assert(Value2VPValue.
count(V) &&
"Value does not exist in VPlan");
1700 return Value2VPValue[V];
1704 assert(V &&
"Trying to get or add the VPValue of a null Value");
1705 if (!Value2VPValue.
count(V))
1748 unsigned TabWidth = 2;
1759 void bumpIndent(
int b) { Indent = std::string((
Depth +=
b) * TabWidth,
' '); }
1775 unsigned getOrCreateBID(
const VPBlockBase *Block) {
1776 return BlockID.
count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
1785 const Twine &Label);
1799 VPlanPrinter::printAsIngredient(OS,
I.V);
1825 "Can't insert new block with successors.");
1843 "Can't insert IfTrue with successors.");
1845 "Can't insert IfFalse with successors.");
1859 "Can't connect two block with different parents");
1861 "Blocks can't have more than two successors.");
1862 From->appendSuccessor(To);
1863 To->appendPredecessor(
From);
1869 assert(To &&
"Successor to disconnect is null.");
1870 From->removeSuccessor(To);
1871 To->removePredecessor(
From);
1878 FromBlock->
getParent() &&
"Must be in same region");
1881 if (!FromLoop || !ToLoop || FromLoop != ToLoop)
1892 return ParentVPL->isLoopLatch(Block);
1934 for (
auto &
I : InterleaveGroupMap)
1936 for (
auto *Ptr : DelSet)
1945 return InterleaveGroupMap.
lookup(Instr);
1952 enum class OpMode {
Failed, Load, Opcode };
1956 struct BundleDenseMapInfo {
1958 return {
reinterpret_cast<VPValue *
>(-1)};
1962 return {
reinterpret_cast<VPValue *
>(-2)};
1986 bool CompletelySLP =
true;
1989 unsigned WidestBundleBits = 0;
1991 using MultiNodeOpTy =
1992 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
2001 bool MultiNodeActive =
false;
2019 std::pair<OpMode, VPValue *> getBest(OpMode
Mode,
VPValue *Last,
2043 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
VPBlockBase & front() const
unsigned getOpcode() const
~VPWidenGEPRecipe() override=default
VFRange(const ElementCount &Start, const ElementCount &End)
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPBlockBase * getSinglePredecessor() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCastValue()
Returns the cast VPValue, if one is attached, or nullptr otherwise.
VPValue * getMask() const
Return the mask used by this recipe.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(struct VPTransformState *State)
Generate the IR code for this VPlan.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
static NodeRef getEntryNode(NodeRef N)
~VPWidenSelectRecipe() override=default
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPIteration(unsigned Part, unsigned Lane)
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit, const std::string &Name="", bool IsReplicator=false)
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
This class represents lattice values for constants.
iterator erase(const_iterator CI)
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
~VPWidenCallRecipe() override=default
Hold analysis information for every loop detected by VPLoopInfo.
static NodeRef getEntryNode(NodeRef N)
VPlanIngredient(const Value *V)
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask)
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBlockBase * getEnclosingBlockWithPredecessors()
const VPRecipeBase & back() const
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
const std::string & getName() const
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Represents a single loop in the control flow graph.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
A recipe for handling all phi nodes except for integer and FP inductions.
const VPBasicBlock * getExitBasicBlock() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &OS) const
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
~VPWidenIntOrFpInductionRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const_iterator end() const
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
Create a new VPWidenPHIRecipe for the reduction Phi described by RdxDesc.
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
An ilist node that can access its parent list.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
VPBlockBase(const unsigned char SC, const std::string &N)
static nodes_iterator nodes_begin(GraphRef N)
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask)
static ChildIteratorType child_end(NodeRef N)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
base_list_type::const_reverse_iterator const_reverse_iterator
~VPInterleaveRecipe() override=default
const Instruction * getUnderlyingInstr() const
bool erase(const KeyT &Val)
void execute(VPTransformState &State) override
Generate the phi/select nodes.
virtual ~VPBlockBase()=default
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
bool isFirstIteration() const
std::unique_ptr< VPlan > VPlanPtr
const_iterator end(StringRef path)
Get end iterator over path.
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
static NodeRef getEntryNode(Inverse< GraphRef > N)
const VPBlocksTy & getHierarchicalPredecessors()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Class that provides utilities for VPBlockBases in VPlan.
operand_iterator op_begin()
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenPHIRecipe() override=default
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
A recipe for widening Call instructions.
bool isLoopLatch(const BlockT *BB) const
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
const VPBlockBase * getEntry() const
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
friend raw_ostream & operator<<(raw_ostream &OS, const VPlan &Plan)
The group of interleaved loads/stores sharing the same stride and close to each other.
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
void execute(VPTransformState &State) override
Generate the phi/select nodes.
virtual void execute(struct VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, Instruction *Cast, TruncInst *Trunc=nullptr)
void setPredicate(VPValue *Pred)
Set the block's predicate.
const InterleaveGroup< Instruction > * getInterleaveGroup()
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
LLVM Basic Block Representation.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
This class augments a recipe with a set of VPValues defined by the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
print alias Alias Set Printer
RecipeListTy::iterator iterator
Instruction iterators...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
bool hasVF(ElementCount VF)
static bool classof(const VPValue *V)
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
This is a concrete Recipe that models a single VPlan-level instruction.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
testing::Matcher< const detail::ErrorHolder & > Failed()
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
static bool classof(const VPValue *V)
VPBlocksTy & getSuccessors()
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
VPValue * getVPValue(Value *V)
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
static ChildIteratorType child_end(NodeRef N)
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
VPBlocksTy & getPredecessors()
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
static nodes_iterator nodes_begin(GraphRef N)
~VPReplicateRecipe() override=default
Drive the analysis of interleaved memory accesses in the loop.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
This class implements an extremely fast bulk output stream that can only output to a stream.
~VPRegionBlock() override
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void addExternalDef(VPValue *VPVal)
Add VPVal to the pool of external definitions if it's not already in the pool.
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
const std::string & getName() const
A Recipe for widening the canonical induction variable of the vector loop.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
static ChildIteratorType child_begin(NodeRef N)
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
~VPWidenCanonicalIVRecipe() override=default
unsigned getNumOperands() const
base_list_type::reverse_iterator reverse_iterator
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask)
VPValue * getAddr() const
Return the address accessed by this recipe.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isStore() const
Returns true if this recipe is a store.
const VPBlocksTy & getPredecessors() const
std::string str() const
Return the twine contents as a std::string.
mir Rename Register Operands
static NodeRef getEntryNode(Inverse< NodeRef > B)
static ChildIteratorType child_begin(NodeRef N)
VPValue * getAddr() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
void clearPredecessors()
Remove all the predecessor of this block.
size_t getNumSuccessors() const
virtual void execute(struct VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPValue * getMask() const
Return the mask used by this recipe.
const VPBasicBlock * getParent() const
const_reverse_iterator rbegin() const
void setName(const Twine &newName)
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
const VPBlocksTy & getHierarchicalSuccessors()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
~VPPredInstPHIRecipe() override=default
VPBranchOnMaskRecipe(VPValue *BlockInMask)
An instruction for storing to memory.
VPValue * getCondOp() const
The VPValue of the condition for the block.
void clearSuccessors()
Remove all the successors of this block and set to null its condition bit.
VPBlockBase * getSingleHierarchicalSuccessor()
VPBlockBase * getSingleHierarchicalPredecessor()
void execute(VPTransformState &State) override
Generate the gep nodes.
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
void setExit(VPBlockBase *ExitBlock)
Set ExitBlock as the exit VPBlockBase of this VPRegionBlock.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
void execute(VPTransformState &State) override
Generate the instruction.
uint32_t getFactor() const
This class represents a truncation of integer types.
base_list_type::iterator iterator
VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
unsigned getVPBlockID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
static ChildIteratorType child_begin(NodeRef N)
VPInstruction(unsigned Opcode, ArrayRef< VPInstruction * > Operands)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
size_t getNumPredecessors() const
void removeVPValueFor(Value *V)
void addVPValue(Value *V)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenCanonicalIVRecipe()
VPValue * getStoredValue() const
Return the address accessed by this recipe.
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector and ...
static NodeRef getEntryNode(GraphRef N)
static nodes_iterator nodes_end(GraphRef N)
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands)
A recipe for handling GEP instructions.
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
This class represents the LLVM 'select' instruction.
@ BasicBlock
Various leaf nodes.
print Print MemDeps of function
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, bool IsPredicated=false)
A Recipe for widening load/store operations.
VPInstruction * clone() const
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
A recipe for widening select instructions.
static nodes_iterator nodes_end(GraphRef N)
static bool classof(const VPDef *R)
Method to support type inquiry through isa, cast, and dyn_cast.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void printAsOperand(raw_ostream &OS, bool PrintType) const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
VPBlockBase * getSingleSuccessor() const
Analysis the ScalarEvolution expression for r is this
void addOperand(VPValue *Operand)
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPBasicBlock * getParent()
static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock, VPLoopInfo *VPLI)
Count and return the number of succesors of PredBlock excluding any backedges.
A recipe for generating conditional branches on the bits of a mask.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
An instruction for reading from memory.
friend class VPInstruction
const VPBasicBlock * getEntryBasicBlock() const
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
void setUnderlyingValue(Value *Val)
void setAlsoPack(bool Pack)
iterator_range< df_iterator< T > > depth_first(const T &G)
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
void setCondBit(VPValue *CV)
Set the condition bit selecting the successor.
void addVPValue(Value *V, VPValue *VPV)
static nodes_iterator nodes_end(GraphRef N)
void setParent(VPRegionBlock *P)
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
const_reverse_iterator rend() const
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
static bool classof(const VPDef *Recipe)
Method to support type inquiry through isa, cast, and dyn_cast.
static bool isBackEdge(const VPBlockBase *FromBlock, const VPBlockBase *ToBlock, const VPLoopInfo *VPLI)
Returns true if the edge FromBlock -> ToBlock is a back-edge.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
auto map_range(ContainerTy &&C, FuncTy F)
const_iterator begin() const
const VPBlockBase * getExit() const
virtual ~VPRecipeBase()=default
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands, Loop *OrigLoop)
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
VPRegionBlock * getParent()
reverse_iterator rbegin()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Iterator for intrusive lists based on ilist_node.
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPBlockBase * setEntry(VPBlockBase *Block)
BlockT * getHeader() const
~VPReductionRecipe() override=default
VPValue * getOrAddVPValue(Value *V)
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
This class provides computation of slot numbers for LLVM Assembly writing.
void setUnderlyingInstr(Instruction *I)
void setName(const Twine &newName)
static bool blockIsLoopLatch(const VPBlockBase *Block, const VPLoopInfo *VPLInfo)
Returns true if Block is a loop latch.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPLoopInfo & getVPLoopInfo()
Return the VPLoopInfo analysis for this VPlan.
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
iterator insert(iterator where, pointer New)
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
VPlan(VPBlockBase *Entry=nullptr)
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
const VPRecipeBase & front() const
iterator begin()
Recipe iterator methods.
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
void execute(VPTransformState &State) override
Generate the wide load/store.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPValue * getOperand(unsigned N) const
base_list_type::const_iterator const_iterator
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
A range adaptor for a pair of iterators.
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands)
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
const VPRegionBlock * getParent() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
A SetVector that performs no allocations if smaller than a certain size.
static bool classof(const VPDef *D)
const VPLoopInfo & getVPLoopInfo() const
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
This class represents a function call, abstracting a target machine's calling convention.
unsigned getVPValueID() const
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands, bool InvariantCond)
VPValue * getMask() const
Return the mask used by this recipe.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
BlockVerifier::State From
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const TruncInst * getTruncInst() const
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
static bool classof(const VPValue *V)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
const VPBlocksTy & getSuccessors() const
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
~VPWidenRecipe() override=default
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void dump() const
Print the VPInstruction to dbgs() (for debugging).
VPValue * getVPValue(unsigned I=0)
Returns the VPValue with index I defined by the VPDef.
static ChildIteratorType child_end(NodeRef N)
VPValue * getStartValue()
Returns the start value of the phi, if it is a reduction.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
static nodes_iterator nodes_begin(GraphRef N)
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
~VPInterleavedAccessInfo()
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
LLVM Value Representation.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
static NodeRef getEntryNode(GraphRef N)
VPWidenPHIRecipe(PHINode *Phi)
Create a VPWidenPHIRecipe for Phi.
void dump() const
Dump the plan to stderr (for debugging).
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStartValue()
Returns the start value of the induction.
VPlanPrinter prints a given VPlan to a given output stream.
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
A Use represents the edge between a Value definition and its users.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
void addVF(ElementCount VF)