Go to the documentation of this file.
24 GetIntOrFpInductionDescriptor,
27 auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
32 if (
Base->getNumPredecessors() == 0 ||
Base->getNumSuccessors() == 0)
38 VPValue *VPV = Ingredient.getVPSingleValue();
40 if (DeadInstructions.
count(Inst)) {
43 Ingredient.eraseFromParent();
48 if (
auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
49 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
50 if (
const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
51 VPValue *Start = Plan->getOrAddVPValue(II->getStartValue());
57 Plan->addVPValue(Phi, VPPhi);
61 assert(isa<VPInstruction>(&Ingredient) &&
62 "only VPInstructions expected here");
63 assert(!isa<PHINode>(Inst) &&
"phis should be handled above");
68 nullptr ,
false ,
false );
72 Plan->getOrAddVPValue(
Store->getValueOperand()),
nullptr ,
76 GEP, Plan->mapToVPValues(
GEP->operands()), OrigLoop);
77 }
else if (
CallInst *CI = dyn_cast<CallInst>(Inst)) {
80 }
else if (
SelectInst *
SI = dyn_cast<SelectInst>(Inst)) {
84 *
SI, Plan->mapToVPValues(
SI->operands()), InvariantCond);
96 "Only recpies with zero or one defined values expected");
97 Ingredient.eraseFromParent();
98 Plan->removeVPValueFor(Inst);
100 Plan->addVPValue(Inst,
Def);
109 bool Changed =
false;
113 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
114 for (
auto &Recipe : *VPBB) {
115 auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
116 if (!RepR || !RepR->isPredicated())
119 WorkList.
insert(std::make_pair(RepR->getParent(),
Op));
124 while (!WorkList.
empty()) {
128 auto *SinkCandidate = dyn_cast_or_null<VPReplicateRecipe>(
C->Def);
129 if (!SinkCandidate || SinkCandidate->isUniform() ||
130 SinkCandidate->getParent() == SinkTo ||
131 SinkCandidate->mayHaveSideEffects() ||
132 SinkCandidate->mayReadOrWriteMemory())
135 bool NeedsDuplicating =
false;
141 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
142 SinkCandidate](
VPUser *U) {
143 auto *UI = dyn_cast<VPRecipeBase>(U);
146 if (UI->getParent() == SinkTo)
148 auto *WidenI = dyn_cast<VPWidenMemoryInstructionRecipe>(UI);
149 if (WidenI && WidenI->getAddr() == SinkCandidate) {
150 NeedsDuplicating =
true;
155 if (!
all_of(SinkCandidate->users(), CanSinkWithUser))
158 if (NeedsDuplicating) {
159 Instruction *
I = cast<Instruction>(SinkCandidate->getUnderlyingValue());
164 Clone->insertBefore(SinkCandidate);
166 for (
auto *U :
Users) {
167 auto *UI = cast<VPRecipeBase>(U);
168 if (UI->getParent() == SinkTo)
171 for (
unsigned Idx = 0; Idx != UI->getNumOperands(); Idx++) {
172 if (UI->getOperand(Idx) != SinkCandidate)
174 UI->setOperand(Idx, Clone);
179 for (
VPValue *
Op : SinkCandidate->operands())
180 WorkList.
insert(std::make_pair(SinkTo,
Op));
189 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
190 if (!EntryBB || EntryBB->size() != 1 ||
191 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
194 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
199 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
200 if (EntryBB->getNumSuccessors() != 2)
203 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
204 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
205 if (!Succ0 || !Succ1)
208 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
210 if (Succ0->getSingleSuccessor() == Succ1)
212 if (Succ1->getSingleSuccessor() == Succ0)
219 bool Changed =
false;
224 VPBlockUtils::blocksOnly<VPRegionBlock>(
depth_first(
231 if (DeletedRegions.
contains(Region1))
233 auto *MiddleBasicBlock =
234 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
235 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
239 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
245 if (!Mask1 || Mask1 != Mask2)
249 if (!Then1 || !Then2)
252 assert(Mask1 && Mask2 &&
"both region must have conditions");
270 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
271 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
274 auto *UI = dyn_cast<VPRecipeBase>(U);
275 if (!UI || UI->getParent() != Then2)
277 for (
unsigned I = 0,
E = U->getNumOperands();
I !=
E; ++
I) {
278 if (Phi1ToMoveV != U->getOperand(
I))
280 U->setOperand(
I, PredInst1);
284 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
293 DeletedRegions.
insert(Region1);
303 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
304 if (!
IV ||
IV->getTruncInst())
315 auto &Casts =
IV->getInductionDescriptor().getCastInsts();
319 for (
auto *U : FindMyCast->
users()) {
320 auto *UserCast = cast<VPRecipeBase>(U);
321 if (UserCast->getNumDefinedValues() == 1 &&
322 UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
323 FoundUserCast = UserCast;
337 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
347 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
349 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
350 WidenOriginalIV->getScalarType() != WidenNewIV->
getScalarType())
357 if (WidenOriginalIV->needsVectorIV() ||
372 if (R.mayHaveSideEffects() ||
any_of(R.definedValues(), [](
VPValue *V) {
373 return V->getNumUsers() > 0;
384 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
385 if (!
IV || !
IV->needsScalarIV())
394 IV->getStartValue(), Step, TruncI ? TruncI->
getType() :
nullptr);
399 if (!
IV->needsVectorIV()) {
400 IV->replaceAllUsesWith(Steps);
408 if (!U->usesScalars(
IV))
410 for (
unsigned I = 0,
E = U->getNumOperands();
I !=
E;
I++) {
411 if (U->getOperand(
I) !=
IV)
413 U->setOperand(
I, Steps);
424 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
428 auto I = SCEV2VPV.
insert({ExpR->getSCEV(), ExpR});
431 ExpR->replaceAllUsesWith(
I.first->second);
432 ExpR->eraseFromParent();
This is an optimization pass for GlobalISel generic memory operations.
Canonical scalar induction phi of the vector loop.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Represents a single loop in the control flow graph.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
ReachingDefAnalysis InstSet & ToRemove
The main scalar evolution driver.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
std::unique_ptr< VPlan > VPlanPtr
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
A recipe for widening Call instructions.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
A struct for saving information about induction variables.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
(vector float) vec_cmpeq(*A, *B) C
void replaceAllUsesWith(VPValue *New)
A Recipe for widening the canonical induction variable of the vector loop.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
bool empty() const
Determine if the SetVector is empty or not.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
An efficient, type-erasing, non-owning reference to a callable.
An instruction for storing to memory.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
StandardInstrumentations SI(Debug, VerifyEach)
A recipe for handling GEP instructions.
This class represents the LLVM 'select' instruction.
A Recipe for widening load/store operations.
A recipe for widening select instructions.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
bool insert(const value_type &X)
Insert a new element into the SetVector.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
VPBlockBase * getSingleSuccessor() const
Type * getType() const
All values are typed, get the type of this value.
An instruction for reading from memory.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
const VPBasicBlock * getEntryBasicBlock() const
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
iterator_range< df_iterator< T > > depth_first(const T &G)
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
const Type * getScalarType() const
Returns the scalar type of the induction.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
iv Induction Variable Users
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
static const uint32_t IV[8]
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
This class represents a function call, abstracting a target machine's calling convention.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
A vector that has set insertion semantics.
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
LLVM_NODISCARD T pop_back_val()
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.