33 GetIntOrFpInductionDescriptor,
38 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
40 auto EndIter = Term ? Term->getIterator() : VPBB->end();
45 VPValue *VPV = Ingredient.getVPSingleValue();
49 if (
auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
50 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
51 if (
const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
52 VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
57 Plan->addVPValue(Phi, VPPhi);
61 assert(isa<VPInstruction>(&Ingredient) &&
62 "only VPInstructions expected here");
63 assert(!isa<PHINode>(Inst) &&
"phis should be handled above");
65 if (
LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
67 *Load, Ingredient.getOperand(0),
nullptr ,
69 }
else if (
StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
71 *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
72 nullptr ,
false ,
false );
75 }
else if (
CallInst *CI = dyn_cast<CallInst>(Inst)) {
79 }
else if (
SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
81 }
else if (
auto *CI = dyn_cast<CastInst>(Inst)) {
83 CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI);
94 "Only recpies with zero or one defined values expected");
95 Ingredient.eraseFromParent();
102 bool Changed =
false;
106 for (
VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
113 for (
auto &Recipe : *VPBB) {
115 if (
auto *Def =
Op->getDefiningRecipe())
116 WorkList.
insert(std::make_pair(VPBB, Def));
122 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
125 std::tie(SinkTo, SinkCandidate) = WorkList[
I];
126 if (SinkCandidate->
getParent() == SinkTo ||
130 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
131 if (!ScalarVFOnly && RepR->isUniform())
133 }
else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
136 bool NeedsDuplicating =
false;
141 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
142 SinkCandidate](
VPUser *U) {
143 auto *UI = dyn_cast<VPRecipeBase>(U);
146 if (UI->getParent() == SinkTo)
151 return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
156 if (NeedsDuplicating) {
160 cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
164 Clone->insertBefore(SinkCandidate);
166 auto *UI = cast<VPRecipeBase>(U);
167 if (UI->getParent() == SinkTo)
170 for (
unsigned Idx = 0;
Idx != UI->getNumOperands();
Idx++) {
173 UI->setOperand(
Idx, Clone);
179 if (
auto *Def =
Op->getDefiningRecipe())
180 WorkList.
insert(std::make_pair(SinkTo, Def));
189 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
190 if (!EntryBB || EntryBB->size() != 1 ||
191 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
194 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
199 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
200 if (EntryBB->getNumSuccessors() != 2)
203 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
204 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
205 if (!Succ0 || !Succ1)
208 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
210 if (Succ0->getSingleSuccessor() == Succ1)
212 if (Succ1->getSingleSuccessor() == Succ0)
227 for (
VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
229 if (!Region1->isReplicator())
231 auto *MiddleBasicBlock =
232 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
233 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
237 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
238 if (!Region2 || !Region2->isReplicator())
243 if (!Mask1 || Mask1 != Mask2)
246 assert(Mask1 && Mask2 &&
"both region must have conditions");
252 if (DeletedRegions.
contains(Region1))
254 auto *MiddleBasicBlock = cast<VPBasicBlock>(Region1->getSingleSuccessor());
255 auto *Region2 = cast<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
259 if (!Then1 || !Then2)
278 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
279 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
281 auto *UI = dyn_cast<VPRecipeBase>(U);
282 if (!UI || UI->getParent() != Then2)
284 for (
unsigned I = 0,
E = U->getNumOperands();
I !=
E; ++
I) {
285 if (Phi1ToMoveV != U->getOperand(
I))
287 U->setOperand(
I, PredInst1);
291 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
300 DeletedRegions.
insert(Region1);
305 return !DeletedRegions.
empty();
312 std::string RegionName = (
Twine(
"pred.") + Instr->getOpcodeName()).str();
313 assert(Instr->getParent() &&
"Predicated instruction not in any basic block");
314 auto *BlockInMask = PredRecipe->
getMask();
346 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
349 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
350 if (RepR->isPredicated())
376 bool ShouldSimplify =
true;
377 while (ShouldSimplify) {
380 ShouldSimplify |= VPlanTransforms::mergeBlocksIntoPredecessors(Plan);
383bool VPlanTransforms::mergeBlocksIntoPredecessors(
VPlan &Plan) {
385 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
388 dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
389 if (PredVPBB && PredVPBB->getNumSuccessors() == 1)
394 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(VPBB->getSinglePredecessor());
396 R.moveBefore(*PredVPBB, PredVPBB->
end());
398 auto *ParentRegion = cast_or_null<VPRegionBlock>(VPBB->getParent());
399 if (ParentRegion && ParentRegion->getExiting() == VPBB)
400 ParentRegion->setExiting(PredVPBB);
401 for (
auto *Succ :
to_vector(VPBB->successors())) {
407 return !WorkList.empty();
410void VPlanTransforms::removeRedundantInductionCasts(
VPlan &Plan) {
412 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
413 if (!
IV ||
IV->getTruncInst())
424 auto &Casts =
IV->getInductionDescriptor().getCastInsts();
428 for (
auto *U : FindMyCast->
users()) {
429 auto *UserCast = cast<VPRecipeBase>(U);
430 if (UserCast->getNumDefinedValues() == 1 &&
431 UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
432 FoundUserCast = UserCast;
442void VPlanTransforms::removeRedundantCanonicalIVs(
VPlan &Plan) {
446 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
456 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
458 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
459 WidenOriginalIV->getScalarType() != WidenNewIV->
getScalarType())
466 if (
any_of(WidenOriginalIV->users(),
467 [WidenOriginalIV](
VPUser *U) {
468 return !U->usesScalars(WidenOriginalIV);
478void VPlanTransforms::removeDeadRecipes(
VPlan &Plan) {
488 [](
VPValue *V) { return V->getNumUsers(); }))
493 auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
494 bool IsConditionalAssume =
495 RepR && RepR->isPredicated() &&
496 match(RepR->getUnderlyingInstr(), m_Intrinsic<Intrinsic::assume>());
497 if (
R.mayHaveSideEffects() && !IsConditionalAssume)
514 if (!CanonicalIV->
isCanonical(
ID.getKind(), StartV, Step, TruncTy)) {
516 TruncI ? TruncI->
getType() :
nullptr);
521 HeaderVPBB->
insert(Steps, IP);
530 auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
533 if (HasOnlyVectorVFs &&
none_of(WideIV->users(), [WideIV](
VPUser *U) {
534 return U->usesScalars(WideIV);
540 Plan,
ID, SE, WideIV->getTruncInst(), WideIV->getPHINode()->getType(),
541 WideIV->getStartValue(), WideIV->getStepValue());
547 if (HasOnlyVectorVFs && !
U->usesScalars(WideIV))
549 for (
unsigned I = 0,
E =
U->getNumOperands();
I !=
E;
I++) {
550 if (
U->getOperand(
I) != WideIV)
552 U->setOperand(
I, Steps);
558void VPlanTransforms::removeRedundantExpandSCEVRecipes(
VPlan &Plan) {
563 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
567 auto I = SCEV2VPV.
insert({ExpR->getSCEV(), ExpR});
570 ExpR->replaceAllUsesWith(
I.first->second);
571 ExpR->eraseFromParent();
576 VPInstruction *Not = dyn_cast<VPInstruction>(Term->getOperand(0));
587 assert(Plan.
hasVF(BestVF) &&
"BestVF is not available in Plan");
588 assert(Plan.
hasUF(BestUF) &&
"BestUF is not available in Plan");
591 auto *Term = dyn_cast<VPInstruction>(&ExitingVPBB->
back());
608 if (TripCount->
isZero() ||
616 Term->eraseFromParent();
627 auto *
Region = dyn_cast_or_null<VPRegionBlock>(R->getParent()->getParent());
630 Region->getNumPredecessors() == 1 &&
"Expected SESE region!");
631 assert(R->getParent()->size() == 1 &&
632 "A recipe in an original replicator region must be the only "
633 "recipe in its block");
646 for (
auto &R : *
A->getParent()) {
656 if (ParentA == ParentB)
657 return LocalComesBefore(
A,
B);
660 "No replicate regions expected at this point");
662 "No replicate regions expected at this point");
677 auto TryToPushSinkCandidate = [&](
VPRecipeBase *SinkCandidate) {
680 if (SinkCandidate == Previous)
683 if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
684 !Seen.
insert(SinkCandidate).second ||
688 if (SinkCandidate->mayHaveSideEffects())
697 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
700 "only recipes with a single defined value expected");
703 if (
auto *R = dyn_cast<VPRecipeBase>(
User))
704 if (!TryToPushSinkCandidate(R))
716 if (SinkCandidate == FOR)
719 SinkCandidate->moveAfter(Previous);
720 Previous = SinkCandidate;
733 if (
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
738 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
741 while (
auto *PrevPhi =
742 dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
743 assert(PrevPhi->getParent() == FOR->getParent());
745 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
754 if (isa<VPHeaderPHIRecipe>(Previous))
759 auto *RecurSplice = cast<VPInstruction>(
761 {FOR, FOR->getBackedgeValue()}));
763 FOR->replaceAllUsesWith(RecurSplice);
766 RecurSplice->setOperand(0, FOR);
774 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
785 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
787 if (
auto *RecWithFlags =
789 RecWithFlags->dropPoisonGeneratingFlags();
793 auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
796 for (
VPValue *V : UserRecipe->definedValues())
807 auto *
C = dyn_cast<ConstantInt>(V->getLiveInIRValue());
808 return C &&
C->isOne();
813 if (
auto *WidenR = dyn_cast<VPWidenRecipe>(&R))
814 return WidenR->getUnderlyingInstr()->getOpcode();
815 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R))
816 return RepR->getUnderlyingInstr()->getOpcode();
817 if (
auto *VPI = dyn_cast<VPInstruction>(&R))
818 return VPI->getOpcode();
825 if (Opcode == Instruction::Mul) {
829 return R.getVPSingleValue()->replaceAllUsesWith(
B);
831 return R.getVPSingleValue()->replaceAllUsesWith(
A);
839 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
847 removeRedundantCanonicalIVs(Plan);
848 removeRedundantInductionCasts(Plan);
850 optimizeInductions(Plan, SE);
852 removeDeadRecipes(Plan);
856 removeRedundantExpandSCEVRecipes(Plan);
857 mergeBlocksIntoPredecessors(Plan);
898 VPValue *StartV = CanonicalIVPHI->getStartValue();
900 auto *CanonicalIVIncrement =
901 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
904 CanonicalIVIncrement->dropPoisonGeneratingFlags();
905 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
915 VPValue *TripCount, *IncrementValue;
920 IncrementValue = CanonicalIVIncrement;
926 IncrementValue = CanonicalIVPHI;
930 auto *EntryIncrement =
Builder.createOverflowingOp(
937 DL,
"active.lane.mask.entry");
942 LaneMaskPhi->insertAfter(CanonicalIVPHI);
947 Builder.setInsertPoint(OriginalTerminator);
948 auto *InLoopIncrement =
950 {IncrementValue}, {
false,
false},
DL);
952 {InLoopIncrement, TripCount},
DL,
953 "active.lane.mask.next");
954 LaneMaskPhi->addOperand(ALM);
958 auto *NotMask =
Builder.createNot(ALM,
DL);
965 VPlan &Plan,
bool UseActiveLaneMaskForControlFlow,
968 UseActiveLaneMaskForControlFlow &&
969 "DataAndControlFlowWithoutRuntimeCheck implies "
970 "UseActiveLaneMaskForControlFlow");
972 auto FoundWidenCanonicalIVUser =
974 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
975 assert(FoundWidenCanonicalIVUser &&
976 "Must have widened canonical IV when tail folding!");
977 auto *WideCanonicalIV =
978 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
980 if (UseActiveLaneMaskForControlFlow) {
986 nullptr,
"active.lane.mask");
995 auto *CompareToReplace = dyn_cast<VPInstruction>(U);
996 if (!CompareToReplace ||
997 CompareToReplace->getOpcode() != Instruction::ICmp ||
999 CompareToReplace->getOperand(1) != BTC)
1002 assert(CompareToReplace->getOperand(0) == WideCanonicalIV &&
1003 "WidenCanonicalIV must be the first operand of the compare");
1005 CompareToReplace->eraseFromParent();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
ReachingDefAnalysis InstSet & ToRemove
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isConstantOne(const Value *Val)
isConstantOne - Return true only if val is constant int 1
iv Induction Variable Users
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static const uint32_t IV[8]
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Core dominator tree base class.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
static constexpr ElementCount getFixed(ScalarTy MinVal)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
A struct for saving information about induction variables.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
RecurKind getRecurrenceKind() const
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
const SCEV * getConstant(ConstantInt *V)
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
VPlan-based builder utility analogous to IRBuilder.
Canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
This is a concrete Recipe that models a single VPlan-level instruction.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
unsigned getOpcode() const
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
VPBasicBlock * getParent()
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
operand_iterator op_end()
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void replaceAllUsesWith(VPValue *New)
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
A recipe for widening Call instructions.
A Recipe for widening the canonical induction variable of the vector loop.
const Type * getScalarType() const
Returns the scalar type of the induction.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A Recipe for widening load/store operations.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPValue * getTripCount() const
The trip count of the original loop.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
VPValue * getVPValueOrAddLiveIn(Value *V)
Gets the VPValue for V or adds a new live-in (if none exists yet) for V.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
bool hasScalarVFOnly() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
A recipe for handling first-order recurrence phis.
A recipe for widening select instructions.